framework,version,device,op_name,kernel_source,m,k,quant_dtype,latency
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,16384,fp8,0.7063210805257161
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,12288,fp8,0.522001075744629
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,10240,fp8,0.43215465545654297
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,8192,fp8,0.3424597422281901
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,7168,fp8,0.29757439295450844
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,6144,fp8,0.2540544033050537
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,5120,fp8,0.211080535252889
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,4096,fp8,0.08772266705830892
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,3584,fp8,0.039389868577321366
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,3072,fp8,0.034577067693074545
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,2560,fp8,0.028842665751775104
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,2048,fp8,0.023413334290186563
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,1536,fp8,0.01802240014076233
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,1024,fp8,0.012492799758911132
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,768,fp8,0.009830400347709656
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,512,fp8,0.007099733253320058
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,256,fp8,0.004300799965858459
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,128,fp8,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,64,fp8,0.0022869333624839783
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,32,fp8,0.0020810666183630625
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8192,65536,fp8,3.1265792846679688
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,16384,fp8,0.34355198542277016
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,12288,fp8,0.2540544033050537
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,10240,fp8,0.21060266494750976
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,8192,fp8,0.08611839612325033
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,7168,fp8,0.03935360113779704
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,6144,fp8,0.034167468547821045
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,5120,fp8,0.028842665751775104
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,4096,fp8,0.023478400707244874
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,3584,fp8,0.020753065745035805
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,3072,fp8,0.01798080007235209
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,2560,fp8,0.015223466356595359
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,2048,fp8,0.012526933352152506
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,1536,fp8,0.009788800279299419
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,1024,fp8,0.007091199855009715
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,768,fp8,0.005761066575845083
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,512,fp8,0.004334933559099833
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,256,fp8,0.002935466667016347
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,128,fp8,0.002218666672706604
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,64,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,32,fp8,0.001911466692884763
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4096,65536,fp8,1.4801920572916667
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,16384,fp8,0.06331733465194703
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,12288,fp8,0.03413333495457967
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,10240,fp8,0.028842665751775104
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,8192,fp8,0.023483733336130776
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,7168,fp8,0.020677334070205687
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,6144,fp8,0.01802240014076233
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,5120,fp8,0.01518933375676473
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,4096,fp8,0.012492799758911132
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,3584,fp8,0.011161599556605022
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,3072,fp8,0.009830400347709656
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,2560,fp8,0.008396800359090168
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,2048,fp8,0.007096533477306366
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,1536,fp8,0.005734399954477946
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,1024,fp8,0.004334933559099833
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,768,fp8,0.00365226666132609
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,512,fp8,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,256,fp8,0.002277333289384842
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,128,fp8,0.002074666693806648
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,64,fp8,0.001911466692884763
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,32,fp8,0.0018677332748969395
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2048,65536,fp8,0.7070720036824544
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,16384,fp8,0.023514666159947715
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,12288,fp8,0.01791999936103821
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,10240,fp8,0.015185067057609558
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,8192,fp8,0.012492799758911132
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,7168,fp8,0.011161599556605022
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,6144,fp8,0.009796266754468281
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,5120,fp8,0.008396800359090168
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,4096,fp8,0.007097599903742473
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,3584,fp8,0.006382933259010315
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,3072,fp8,0.005734399954477946
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,2560,fp8,0.005051733553409576
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,2048,fp8,0.004334933559099833
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,1536,fp8,0.003618133316437403
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,1024,fp8,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,768,fp8,0.0025941332181294756
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,512,fp8,0.0022528000175952912
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,256,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,128,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,64,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,32,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1024,65536,fp8,0.34354772567749026
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,16384,fp8,0.01795413295427958
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,12288,fp8,0.013890133301417033
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,10240,fp8,0.01181013286113739
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,8192,fp8,0.009830400347709656
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,7168,fp8,0.008772266904513042
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,6144,fp8,0.007714133461316426
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,5120,fp8,0.006724266707897187
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,4096,fp8,0.005700266857941946
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,3584,fp8,0.0052149335543314615
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,3072,fp8,0.004676266511281332
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,2560,fp8,0.004164266586303711
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,2048,fp8,0.003618133316437403
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,1536,fp8,0.0031061333914597826
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,1024,fp8,0.00262719988822937
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,768,fp8,0.0023552000522613524
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,512,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,256,fp8,0.0019039999693632125
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,128,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,64,fp8,0.0019050666441520054
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,32,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,768,65536,fp8,0.2543274720509847
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,16384,fp8,0.012526933352152506
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,12288,fp8,0.009864532947540283
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,10240,fp8,0.008396800359090168
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,8192,fp8,0.007065600156784058
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,7168,fp8,0.006411733229955037
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,6144,fp8,0.005700266857941946
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,5120,fp8,0.005017599960168203
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,4096,fp8,0.004334933559099833
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,3584,fp8,0.0039594667653242745
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,3072,fp8,0.003618133316437403
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,2560,fp8,0.0033098667860031127
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,2048,fp8,0.002935466667016347
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,1536,fp8,0.0025941332181294756
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,1024,fp8,0.002218666672706604
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,768,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,512,fp8,0.0020394666741291685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,128,fp8,0.0018730666488409042
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,64,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,32,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,512,65536,fp8,0.0524288018544515
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,16384,fp8,0.009796266754468281
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,12288,fp8,0.0076799998680750535
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,10240,fp8,0.006758399804433187
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,8192,fp8,0.005700266857941946
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,7168,fp8,0.005154133339722951
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,6144,fp8,0.004642133414745331
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,5120,fp8,0.004130133241415024
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,4096,fp8,0.0036864000062147772
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,3584,fp8,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,3072,fp8,0.0031061333914597826
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,2560,fp8,0.002792533238728841
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,2048,fp8,0.0025941332181294756
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,1536,fp8,0.0023552000522613524
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,1024,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,768,fp8,0.002077866718173027
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,128,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,64,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,32,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,384,65536,fp8,0.0340992013613383
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,16384,fp8,0.007065600156784058
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,12288,fp8,0.005700266857941946
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,10240,fp8,0.004983466863632202
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,8192,fp8,0.004330666859944662
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,7168,fp8,0.003986133386691412
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,6144,fp8,0.00365226666132609
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,5120,fp8,0.0033013333876927694
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,4096,fp8,0.002935466667016347
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,3584,fp8,0.0027647999425729113
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,3072,fp8,0.002587733417749405
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,2560,fp8,0.002423466742038727
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,2048,fp8,0.0022528000175952912
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,1536,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,1024,fp8,0.0020426666984955473
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,768,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,128,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,64,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,32,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,256,65536,fp8,0.023449599742889404
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,16384,fp8,0.005725866556167603
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,12288,fp8,0.004676266511281332
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,10240,fp8,0.004130133241415024
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,8192,fp8,0.00365226666132609
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,7168,fp8,0.0034133332471052804
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,6144,fp8,0.00314026673634847
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,5120,fp8,0.0027647999425729113
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,4096,fp8,0.002624000112215678
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,3584,fp8,0.002422400067249934
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,3072,fp8,0.0023552000522613524
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,2560,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,2048,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,1536,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,1024,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,768,fp8,0.0021503999829292296
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,512,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,256,fp8,0.001874133323629697
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,128,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,64,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,32,fp8,0.0018719999740521114
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,192,65536,fp8,0.01795413295427958
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,16384,fp8,0.005014400184154511
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,12288,fp8,0.004130133241415024
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,10240,fp8,0.0037205333511034647
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,8192,fp8,0.003310933212439219
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,7168,fp8,0.0031061333914597826
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,6144,fp8,0.0027306665976842242
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,5120,fp8,0.00266239990790685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,4096,fp8,0.002423466742038727
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,3584,fp8,0.0023210667073726653
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,3072,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,2560,fp8,0.00207893339296182
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,2048,fp8,0.002074666693806648
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,1536,fp8,0.0019456000377734502
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,1024,fp8,0.001911466692884763
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,768,fp8,0.0018730666488409042
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,128,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,64,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,32,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,160,65536,fp8,0.015253333250681558
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,16384,fp8,0.004300799965858459
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,12288,fp8,0.0036821333070596062
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,10240,fp8,0.0032768001159032187
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,8192,fp8,0.002935466667016347
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,7168,fp8,0.0027306665976842242
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,6144,fp8,0.002628266563018163
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,5120,fp8,0.00238933339715004
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,4096,fp8,0.0022453332940737406
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,3584,fp8,0.002148266633351644
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,3072,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,2560,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,2048,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,1536,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,1024,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,768,fp8,0.0018751999984184902
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,128,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,64,fp8,0.0018730666488409042
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,32,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,128,65536,fp8,0.012485333283742269
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,16384,fp8,0.00365226666132609
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,12288,fp8,0.003139200061559677
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,10240,fp8,0.0027989332874615988
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,8192,fp8,0.0025941332181294756
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,7168,fp8,0.002423466742038727
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,6144,fp8,0.0023552000522613524
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,5120,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,4096,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,3584,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,3072,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,2560,fp8,0.0019744000087181727
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,2048,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,1536,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,1024,fp8,0.0019456000377734502
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,768,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,128,fp8,0.001874133323629697
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,64,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,32,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,96,65536,fp8,0.009796266754468281
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,16384,fp8,0.003310933212439219
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,12288,fp8,0.0027647999425729113
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,10240,fp8,0.002628266563018163
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,8192,fp8,0.002628266563018163
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,7168,fp8,0.002348800003528595
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,6144,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,5120,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,4096,fp8,0.0020799999435742694
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,3584,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,3072,fp8,0.0019456000377734502
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,2560,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,2048,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,1536,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,1024,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,768,fp8,0.0019061333189407986
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,128,fp8,0.001911466692884763
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,64,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,32,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,80,65536,fp8,0.008362666765848795
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,16384,fp8,0.0029696000119050344
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,12288,fp8,0.0025941332181294756
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,10240,fp8,0.00238933339715004
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,8192,fp8,0.0022463999688625337
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,7168,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,6144,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,5120,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,4096,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,3584,fp8,0.0019050666441520054
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,3072,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,2560,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,2048,fp8,0.001911466692884763
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,1536,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,1024,fp8,0.001874133323629697
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,768,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,256,fp8,0.0018719999740521114
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,128,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,64,fp8,0.0019018666197856266
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,32,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,64,65536,fp8,0.007065600156784058
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,16384,fp8,0.002628266563018163
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,12288,fp8,0.0023552000522613524
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,10240,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,8192,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,7168,fp8,0.002074666693806648
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,6144,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,5120,fp8,0.0019456000377734502
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,4096,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,3584,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,3072,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,2560,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,2048,fp8,0.0018709332992633183
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,1536,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,1024,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,768,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,256,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,128,fp8,0.0019039999693632125
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,64,fp8,0.001874133323629697
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,32,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,48,65536,fp8,0.005726933479309082
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,16384,fp8,0.0022528000175952912
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,12288,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,10240,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,8192,fp8,0.0020768000433842343
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,7168,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,6144,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,5120,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,4096,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,3584,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,3072,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,2560,fp8,0.0019050666441520054
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,2048,fp8,0.0018730666488409042
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,1536,fp8,0.0018751999984184902
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,1024,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,768,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,512,fp8,0.0018730666488409042
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,128,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,64,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,32,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,32,65536,fp8,0.004328533510367076
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,16384,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,12288,fp8,0.0018698666244745255
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,10240,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,8192,fp8,0.0019061333189407986
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,7168,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,6144,fp8,0.001876266673207283
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,5120,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,4096,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,3584,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,3072,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,2560,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,2048,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,1536,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,1024,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,768,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,128,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,64,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,32,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,16,65536,fp8,0.002935466667016347
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,16384,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,12288,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,10240,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,8192,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,7168,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,6144,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,5120,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,4096,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,3584,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,3072,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,2560,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,2048,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,1536,fp8,0.0019456000377734502
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,1024,fp8,0.001911466692884763
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,768,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,512,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,128,fp8,0.00191040001809597
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,64,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,32,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,8,65536,fp8,0.0022528000175952912
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,16384,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,12288,fp8,0.0021162666380405426
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,10240,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,8192,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,7168,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,6144,fp8,0.001876266673207283
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,5120,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,4096,fp8,0.0018730666488409042
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,3584,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,3072,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,2560,fp8,0.001876266673207283
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,2048,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,1536,fp8,0.001876266673207283
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,1024,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,768,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,256,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,128,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,64,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,32,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,4,65536,fp8,0.0020479999482631685
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,16384,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,12288,fp8,0.001876266673207283
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,10240,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,8192,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,7168,fp8,0.001874133323629697
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,6144,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,5120,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,4096,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,3584,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,3072,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,2560,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,2048,fp8,0.001874133323629697
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,1536,fp8,0.001911466692884763
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,1024,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,768,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,512,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,256,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,128,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,64,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,32,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,2,65536,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,16384,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,12288,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,10240,fp8,0.001874133323629697
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,8192,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,7168,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,6144,fp8,0.0020821332931518556
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,5120,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,4096,fp8,0.0018687999496857326
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,3584,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,3072,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,2560,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,2048,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,1536,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,1024,fp8,0.0018773333479960759
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,768,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,512,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,256,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,128,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,64,fp8,0.0018432000031073886
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,32,fp8,0.0018719999740521114
TRTLLM,1.0.0,NVIDIA L40S,scale_matrix,torch_ops,1,65536,fp8,0.0018773333479960759
