framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,1,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,1,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,1,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,1,0.011178666104873022
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,1,0.010922666639089584
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,1,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,1,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,1,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,1,0.010965333630641302
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,1,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,1,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,1,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,1,0.012138667205969492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,1,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,1,0.012138667205969492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,3,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,3,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,3,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,3,0.013514666507641474
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,3,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,1,0.01219733307758967
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,3,0.012725333372751871
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,3,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,3,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,3,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,3,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,3,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,3,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,3,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,3,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,3,0.01091733326514562
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,3,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,7,0.01210133358836174
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,7,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,7,0.011898666620254517
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,7,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,7,0.011045332998037338
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,7,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,7,0.011109333485364914
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,7,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,7,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,7,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,7,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,7,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,7,0.011946666985750198
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,7,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,15,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,15,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,15,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,15,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,15,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,15,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,15,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,15,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,15,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,15,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,7,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,15,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,15,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,15,0.011765333513418833
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,15,0.011007999380429586
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,15,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,15,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,31,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,31,0.011600000162919363
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,31,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,31,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,31,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,31,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,31,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,31,0.011226666470368704
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,31,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,31,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,31,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,31,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,31,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,31,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,31,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,63,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,63,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,63,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,63,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,63,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,63,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,63,0.012223999947309494
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,63,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,63,0.012037333101034164
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,63,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,63,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,63,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,63,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,63,0.011381333072980246
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,63,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,63,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,127,0.011247999966144562
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,127,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,127,0.013503999759753546
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,127,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,127,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,127,0.011674666156371435
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,127,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,127,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,127,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,127,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,127,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,127,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,127,0.011999999483426413
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,127,0.011424000064531961
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,127,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,255,0.012042666474978128
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,255,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,255,0.012128000458081564
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,255,0.012122667084137598
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,255,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,255,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,255,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,255,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,255,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,255,0.013839999834696451
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,255,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,255,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,255,0.011765333513418833
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,255,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,255,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,255,0.012250666817029318
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,511,0.015392000476519266
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,511,0.015413332730531693
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,511,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,511,0.01545599972208341
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,511,0.013525333255529404
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,511,0.01339200014869372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,511,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,511,0.013866666704416275
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,511,0.015237333873907724
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,511,0.014581333845853806
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,511,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,511,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,511,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,511,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,511,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,511,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,1023,0.015578666081031164
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,1023,0.015376000354687372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,1023,0.01515199989080429
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,1023,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,1023,0.015413332730531693
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,1023,0.01340266689658165
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,1023,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,1023,0.013280000537633896
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,1023,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,1023,0.014858666807413101
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,1023,0.015119999647140503
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,1023,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,1023,0.013776000589132309
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,1023,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,1023,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,1023,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,2047,0.0176959993938605
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,2047,0.0174346665541331
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,2047,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,2047,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,2047,0.01471466695268949
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,2047,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,2047,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,2047,0.015935999651749928
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,2047,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,2047,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,2047,0.01360000049074491
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,2047,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,2047,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,2047,0.013855999956528345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,4095,0.019850666324297588
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,2047,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,4095,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,4095,0.01894933357834816
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,4095,0.01717866708834966
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,4095,0.01714666684468587
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,4095,0.016869333883126576
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,4095,0.017621333400408428
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,4095,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,4095,0.019002666076024372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,4095,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,4095,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,4095,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,4095,0.017071999609470367
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,4095,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,4095,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,4095,0.015135999768972397
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,8191,0.027242665489514668
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,8191,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,8191,0.023792001108328503
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,8191,0.01988799994190534
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,8191,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,8191,0.01933866615096728
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,8191,0.021242665747801464
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,8191,0.021253332495689392
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,8191,0.022474666436513264
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,8191,0.019317333896954853
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,8191,0.0191040001809597
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,8191,0.01940800001223882
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,8191,0.017338667064905167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,8191,0.01748266691962878
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,8191,0.017349333812793095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,8191,0.018629333625237148
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,16383,0.037647999823093414
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,16383,0.029232000311215717
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,16383,0.024106666445732117
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,16383,0.025472000241279602
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,16383,0.025685332715511322
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,16383,0.025536000728607178
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,16383,0.025631998976071674
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,16383,0.025936000049114227
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,16383,0.027237333357334137
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,16383,0.02311466634273529
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,16383,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,16383,0.023221333821614582
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,16383,0.02176533391078313
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,16383,0.021082667013009388
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,16383,0.02162666618824005
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,16383,0.02063999945918719
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,32767,0.0367999995748202
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,32767,0.0377813329299291
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,32767,0.031744000812371574
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,32767,0.02848000079393387
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,32767,0.0272533322374026
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,32767,0.030053332448005676
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,32767,0.02804800122976303
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,32767,0.02934933453798294
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,32767,0.03369066615899404
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,32767,0.02784000088771184
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,32767,0.02532800038655599
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,32767,0.02759466568628947
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,32767,0.025978667040665943
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,32767,0.02441066751877467
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,32767,0.02589333305756251
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,32767,0.02569599946339925
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,65535,0.03975466638803482
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,65535,0.042261332273483276
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,65535,0.04450666904449463
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,65535,0.0360959991812706
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,65535,0.03562133262554804
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,65535,0.037205333511034645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,65535,0.035930665830771126
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,65535,0.03648533423741659
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,65535,0.0359946663180987
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,65535,0.03382399926582972
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,65535,0.02972800036271413
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,65535,0.029637334247430164
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,65535,0.027327999472618103
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,65535,0.02941333254178365
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,65535,0.02957333376010259
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,65535,0.027669332921504974
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1,1,1,131071,0.05816000203291575
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1,1,2,131071,0.06332266827424367
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1,1,4,131071,0.07163733243942261
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1,1,8,131071,0.05453333258628845
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1,1,16,131071,0.05433600147565206
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1,1,32,131071,0.05649066468079885
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1,1,64,131071,0.05572799841562907
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1,1,128,131071,0.056287998954455055
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1,1,1,131071,0.04201066493988037
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1,1,2,131071,0.040864000717798867
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1,1,4,131071,0.0421066681543986
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1,1,8,131071,0.03588266670703888
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1,1,16,131071,0.0356480007370313
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1,1,32,131071,0.03401066611210505
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1,1,64,131071,0.03579733272393545
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1,1,128,131071,0.03604800005753835
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,1,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,1,0.01192533348997434
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,1,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,1,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,1,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,1,0.010954666882753372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,1,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,1,0.012661332885424295
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,1,0.013647999614477158
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,1,0.01219733307758967
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,1,0.011493333925803503
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,1,0.012661332885424295
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,1,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,1,0.011616000284751257
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,1,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,1,0.011861333002646765
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,3,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,3,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,3,0.011663999408483505
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,3,0.012602667013804117
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,3,0.012085333466529846
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,3,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,3,0.013408000270525614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,3,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,3,0.012448000411192576
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,3,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,3,0.013424000392357508
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,3,0.011887999872366587
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,3,0.012655999511480331
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,3,0.011706666400035223
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,3,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,3,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,7,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,7,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,7,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,7,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,7,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,7,0.011749333391586939
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,7,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,7,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,7,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,7,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,7,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,7,0.01128000020980835
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,7,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,7,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,7,0.011152000476916632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,7,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,15,0.012527999778588613
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,15,0.011637333780527115
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,15,0.011493333925803503
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,15,0.011839999506870905
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,15,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,15,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,15,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,15,0.011440000186363855
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,15,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,15,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,15,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,15,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,15,0.0116799995303154
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,15,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,15,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,31,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,31,0.01358933374285698
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,31,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,31,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,31,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,31,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,31,0.011855999628702799
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,31,0.01227733368674914
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,31,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,31,0.01139733319481214
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,31,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,31,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,31,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,31,0.01190399999419848
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,31,0.011136000355084738
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,31,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,63,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,63,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,63,0.012954667210578918
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,63,0.01589866727590561
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,63,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,63,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,63,0.012479999413092932
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,63,0.012245333443085352
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,63,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,63,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,63,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,63,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,63,0.011701333026091257
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,63,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,63,0.011114666859308878
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,63,0.011722666521867117
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,127,0.013765333841244379
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,127,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,127,0.011488000551859537
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,127,0.011861333002646765
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,127,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,127,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,127,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,127,0.011312000453472137
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,127,0.011887999872366587
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,127,0.013023999830087027
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,127,0.010992000500361124
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,127,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,127,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,127,0.011445333560307821
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,127,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,127,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,255,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,255,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,255,0.011695999652147293
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,255,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,255,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,255,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,255,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,255,0.01156266654531161
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,255,0.011999999483426413
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,255,0.013525333255529404
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,255,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,255,0.013616000612576803
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,255,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,255,0.012703999876976013
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,255,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,255,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,511,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,511,0.013445333888133367
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,511,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,511,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,511,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,511,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,511,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,511,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,511,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,511,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,511,0.013290667285521826
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,511,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,511,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,511,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,511,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,511,0.013125333935022354
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,1023,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,1023,0.013823999712864557
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,1023,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,1023,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,1023,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,1023,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,1023,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,1023,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,1023,0.01392000044385592
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,1023,0.013658666362365087
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,1023,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,1023,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,1023,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,1023,0.012608000387748083
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,1023,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,1023,0.01293333371480306
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,2047,0.01828266680240631
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,2047,0.016842667013406754
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,2047,0.017360000560681026
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,2047,0.013349333157142004
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,2047,0.017093333105246227
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,2047,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,2047,0.013445333888133367
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,2047,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,2047,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,2047,0.015381333728631338
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,2047,0.015637333194414776
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,2047,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,2047,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,2047,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,2047,0.013429333766301474
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,2047,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,4095,0.023311999936898548
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,4095,0.01924266666173935
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,4095,0.017312000195185345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,4095,0.017279999951521557
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,4095,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,4095,0.017488000293572743
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,4095,0.01706133286158244
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,4095,0.01952533299724261
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,4095,0.017445333302021027
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,4095,0.01783466711640358
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,4095,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,4095,0.017429333180189133
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,4095,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,4095,0.01684800038735072
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,4095,0.015882667154073715
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,4095,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,8191,0.03164266546567281
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,8191,0.025759999950726826
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,8191,0.02223466585079829
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,8191,0.0195573332409064
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,8191,0.019621333728233974
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,8191,0.01961600035429001
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,8191,0.019498666127522785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,8191,0.019573333362738293
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,8191,0.02363733450571696
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,8191,0.0194560003777345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,8191,0.02146133283774058
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,8191,0.019237333287795384
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,8191,0.019440000255902607
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,8191,0.01889066646496455
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,8191,0.017269333203633625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,8191,0.018730666488409042
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,16383,0.02758399893840154
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,16383,0.0336053321758906
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,16383,0.0286613330245018
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,16383,0.02533866713444392
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,16383,0.02553066611289978
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,16383,0.025413334369659424
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,16383,0.023711999257405598
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,16383,0.025519999365011852
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,16383,0.027285332481066387
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,16383,0.025706666211287182
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,16383,0.023258666197458904
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,16383,0.022991999983787537
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,16383,0.02350933353106181
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,16383,0.022672000030676525
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,16383,0.023562667270501454
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,16383,0.02142400046189626
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,32767,0.03389333436886469
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,32767,0.03888533264398575
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,32767,0.04029866556326548
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,32767,0.03201066702604294
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,32767,0.03163733333349228
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,32767,0.03158933420976003
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,32767,0.032442666590213776
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,32767,0.03178133318821589
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,32767,0.02974933385848999
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,32767,0.03390933324893316
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,32767,0.0273333340883255
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,32767,0.02789866675933202
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,32767,0.027301333844661713
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,32767,0.027509334186712902
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,32767,0.025962665677070618
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,32767,0.027477333943049114
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,65535,0.055888002117474876
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,65535,0.06020266811052958
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,65535,0.05983466903368632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,65535,0.054698665936787925
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,65535,0.054416000843048096
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,65535,0.055871998270352684
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,65535,0.05571199953556061
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,65535,0.056421334544817604
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,65535,0.03555733213822047
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,65535,0.039818666875362396
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,65535,0.03456533451875051
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,65535,0.031541332602500916
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,65535,0.032325332363446556
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,65535,0.031871999303499855
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,65535,0.03151999910672506
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,65535,0.03179199993610382
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,2,1,1,131071,0.07595733304818471
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,2,1,2,131071,0.08599467078844707
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,2,1,4,131071,0.09546666344006856
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,2,1,8,131071,0.07898666461308797
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,2,1,16,131071,0.07769600053628285
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,2,1,32,131071,0.08152533570925395
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,2,1,64,131071,0.08157333234945933
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,2,1,128,131071,0.08073066671689351
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,2,1,1,131071,0.05339199801286062
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,2,1,2,131071,0.05754666527112325
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,2,1,4,131071,0.05446400245030721
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,2,1,8,131071,0.04844266672929128
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,2,1,16,131071,0.04783466458320618
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,2,1,64,131071,0.0499839981396993
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,2,1,32,131071,0.0488319993019104
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,2,1,128,131071,0.048581331968307495
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,1,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,1,0.012256000190973282
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,1,0.012304000556468964
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,1,0.011685332904259363
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,1,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,1,0.010869332899649939
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,1,0.01157333329319954
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,1,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,1,0.012906666845083237
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,1,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,1,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,1,0.01228800043463707
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,1,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,1,0.011077333241701126
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,1,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,3,0.013765333841244379
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,3,0.012826666235923767
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,3,0.011674666156371435
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,3,0.01379199946920077
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,3,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,3,0.01138666644692421
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,3,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,3,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,3,0.012144000579913458
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,3,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,3,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,3,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,3,0.011642667154471079
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,3,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,7,0.013797332843144735
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,7,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,7,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,7,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,7,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,7,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,7,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,7,0.013882666826248169
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,7,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,7,0.013594667116800943
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,7,0.012768000364303589
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,7,0.011306667079528173
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,7,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,7,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,7,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,7,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,15,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,15,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,15,0.011834666132926941
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,15,0.012565333396196365
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,15,0.011253333340088526
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,15,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,15,0.011658667276302973
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,15,0.011498666057984034
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,15,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,15,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,15,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,15,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,15,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,15,0.011365332951148352
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,15,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,15,0.01173866664369901
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,31,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,31,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,31,0.011946666985750198
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,31,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,31,0.011472000430027643
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,31,0.011391999820868174
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,31,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,31,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,31,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,31,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,31,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,31,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,31,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,31,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,31,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,31,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,63,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,63,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,63,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,63,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,63,0.014090667168299357
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,63,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,63,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,63,0.013584000368913015
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,63,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,63,0.01116266722480456
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,63,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,63,0.01173866664369901
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,63,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,63,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,63,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,63,0.011296000331640244
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,127,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,127,0.011087999989589056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,127,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,127,0.012069333344697952
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,127,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,127,0.01089599976936976
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,127,0.011301333705584208
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,127,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,127,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,127,0.011349332829316458
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,127,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,127,0.011839999506870905
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,127,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,127,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,127,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,127,0.011626667032639185
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,255,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,255,0.012410666793584824
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,255,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,255,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,255,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,255,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,255,0.011882666498422623
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,255,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,255,0.013487999637921652
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,255,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,255,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,255,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,255,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,255,0.011600000162919363
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,255,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,255,0.011850666254758835
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,511,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,511,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,511,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,511,0.013349333157142004
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,511,0.013061333447694778
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,511,0.01339200014869372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,511,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,511,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,511,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,511,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,511,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,511,0.012752000242471695
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,511,0.01393066719174385
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,511,0.011952000359694162
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,511,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,511,0.012191999703645706
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,1023,0.017664000391960144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,1023,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,1023,0.015573333948850632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,1023,0.01360000049074491
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,1023,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,1023,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,1023,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,1023,0.01351999988158544
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,1023,0.015439999600251516
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,1023,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,1023,0.016949333250522614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,1023,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,1023,0.01246400053302447
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,1023,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,1023,0.012576000144084295
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,1023,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,2047,0.022639999787012737
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,2047,0.017407999684413273
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,2047,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,2047,0.01727466657757759
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,2047,0.014725333700577417
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,2047,0.01551466683546702
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,2047,0.015909332782030106
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,2047,0.015024000157912573
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,2047,0.017423999806245167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,2047,0.01732800031701724
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,2047,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,2047,0.015333333363135656
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,2047,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,2047,0.015029333531856537
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,2047,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,4095,0.030597334106763203
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,4095,0.023760000864664715
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,4095,0.019519999623298645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,4095,0.017557332913080852
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,4095,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,4095,0.017450666675964992
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,4095,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,4095,0.017242666333913803
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,4095,0.021989333132902782
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,4095,0.019573333362738293
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,4095,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,4095,0.017125333348910015
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,4095,0.014970666418472925
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,4095,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,4095,0.01578666642308235
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,4095,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,8191,0.025402667621771496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,8191,0.03269333392381668
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,8191,0.02139200021823247
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,8191,0.025285333395004272
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,8191,0.02111999938885371
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,8191,0.021114667256673176
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,8191,0.02160533269246419
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,8191,0.021141332884629566
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,8191,0.023893333971500397
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,8191,0.023749334116776783
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,8191,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,8191,0.019434666881958645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,8191,0.01924266666173935
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,8191,0.01915733392039935
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,8191,0.019285333653291065
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,8191,0.01754666616519292
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,16383,0.03160000095764796
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,16383,0.03411199897527695
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,16383,0.03800000001986822
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,16383,0.02792533238728841
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,16383,0.02762666592995326
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,16383,0.027098665634791057
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,16383,0.027818667391935985
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,16383,0.027429332335789997
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,16383,0.026528000831604004
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,16383,0.02828266719977061
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,16383,0.025397333006064098
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,16383,0.02421333392461141
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,16383,0.021930667261282604
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,16383,0.02330133318901062
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,16383,0.021744000415007275
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,16383,0.02222399910291036
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,32767,0.04905599852403005
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,32767,0.05436799923578898
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,32767,0.0561653325955073
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,32767,0.05197866757710775
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,32767,0.05041599770387014
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,32767,0.049600000182787575
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,32767,0.050981332858403526
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,32767,0.050330668687820435
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,32767,0.03233066697915395
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,32767,0.03126933425664902
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,32767,0.033626665671666466
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,32767,0.027488000690937042
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,32767,0.026922665536403656
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,32767,0.027402666707833607
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,32767,0.02606933315594991
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,32767,0.02587199956178665
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,65535,0.07228800157705943
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,65535,0.0798773318529129
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,65535,0.09246399998664856
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,65535,0.07588799794514973
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,65535,0.07528000076611836
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,65535,0.07323733468850453
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,65535,0.07534400125344594
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,65535,0.07707199951012929
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,65535,0.05060799916585287
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,65535,0.050442665815353394
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,65535,0.05259199937184652
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,65535,0.04407466451327006
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,65535,0.04372266431649526
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,65535,0.043375998735427856
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,65535,0.044437333941459656
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,65535,0.043621331453323364
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,4,1,1,131071,0.11958400408426921
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,4,1,2,131071,0.130922665198644
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,4,1,4,131071,0.1618133286635081
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,4,1,8,131071,0.12547733386357626
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,4,1,16,131071,0.12306132912635803
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,4,1,32,131071,0.1272053321202596
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,4,1,64,131071,0.12563199798266092
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,4,1,128,131071,0.12213333447774251
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,4,1,1,131071,0.07379733522733052
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,4,1,2,131071,0.07329066594441731
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,4,1,4,131071,0.08080533146858215
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,4,1,8,131071,0.06522666911284129
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,4,1,32,131071,0.06524266799290974
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,4,1,16,131071,0.06530666848023732
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,4,1,64,131071,0.06538133323192596
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,4,1,128,131071,0.06472533444563548
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,1,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,1,0.013807999591032663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,1,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,1,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,1,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,1,0.012821332861979803
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,1,0.011525332927703857
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,1,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,1,0.013466666142145792
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,1,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,1,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,1,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,1,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,1,0.011039999624093374
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,1,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,1,0.011429333438475927
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,3,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,3,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,3,0.013663999736309052
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,3,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,3,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,3,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,3,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,3,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,3,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,3,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,3,0.014463999619086584
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,3,0.012928000340859095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,3,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,3,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,3,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,3,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,7,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,7,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,7,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,7,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,7,0.011482667177915573
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,7,0.011034666250149408
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,7,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,7,0.01301866645614306
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,7,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,7,0.012874666601419449
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,7,0.011855999628702799
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,7,0.01102399950226148
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,7,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,7,0.01098666712641716
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,7,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,7,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,15,0.01381333296497663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,15,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,15,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,15,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,15,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,15,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,15,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,15,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,15,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,15,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,15,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,15,0.011717333147923151
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,15,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,15,0.010981333752473196
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,15,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,15,0.011130666981140772
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,31,0.013514666507641474
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,31,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,31,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,31,0.013536000003417334
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,31,0.011989332735538483
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,31,0.01129066695769628
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,31,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,31,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,31,0.013717333475748697
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,31,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,31,0.011343999455372492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,31,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,31,0.011589333415031433
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,31,0.011055999745925268
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,31,0.011231999844312668
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,31,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,63,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,63,0.012960000584522883
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,63,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,63,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,63,0.013525333255529404
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,63,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,63,0.011893333246310553
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,63,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,63,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,63,0.013845333208640417
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,63,0.012746666868527731
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,63,0.01090666651725769
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,63,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,63,0.011317333827416102
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,63,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,63,0.01145600030819575
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,127,0.013786666095256805
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,127,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,127,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,127,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,127,0.011610666910807291
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,127,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,127,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,127,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,127,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,127,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,127,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,127,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,127,0.01181866725285848
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,127,0.011221333096424738
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,127,0.01180800050497055
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,127,0.011541333049535751
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,255,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,255,0.01310933381319046
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,255,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,255,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,255,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,255,0.011333333949247995
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,255,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,255,0.011418666690587997
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,255,0.014138666292031607
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,255,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,255,0.013818666338920593
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,255,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,255,0.01163200040658315
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,255,0.011546666423479715
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,255,0.01109333336353302
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,255,0.011760000139474869
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,511,0.018800000349680584
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,511,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,511,0.0138026662170887
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,511,0.013418667018413544
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,511,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,511,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,511,0.014080000420411428
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,511,0.015376000354687372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,511,0.01339200014869372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,511,0.013722666849692663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,511,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,511,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,511,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,511,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,511,0.011530666301647821
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,511,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,1023,0.01691199963291486
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,1023,0.021327999730904896
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,1023,0.015642666568358738
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,1023,0.014912000546852747
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,1023,0.014864000181357065
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,1023,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,1023,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,1023,0.014736000448465347
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,1023,0.017632000148296356
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,1023,0.014767999450365702
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,1023,0.01381333296497663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,1023,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,1023,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,1023,0.013295999417702356
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,1023,0.013642666240533194
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,1023,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,2047,0.02754133443037669
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,2047,0.02325333406527837
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,2047,0.01934933289885521
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,2047,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,2047,0.015754666179418564
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,2047,0.015504000087579092
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,2047,0.01580799991885821
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,2047,0.015402667224407196
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,2047,0.022005334496498108
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,2047,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,2047,0.017152000218629837
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,2047,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,2047,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,2047,0.014469332993030548
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,2047,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,2047,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,4095,0.023728000621000927
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,4095,0.023685333629449207
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,4095,0.04067199925581614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,4095,0.01915733392039935
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,4095,0.01912533367673556
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,4095,0.019248000035683315
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,4095,0.01934933289885521
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,4095,0.019173332800467808
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,4095,0.022042666872342426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,4095,0.021136000752449036
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,4095,0.019343999524911244
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,4095,0.01708799973130226
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,4095,0.01706133286158244
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,4095,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,4095,0.015626666446526844
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,4095,0.016927999754746754
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,8191,0.029658667743206024
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,8191,0.029690665503342945
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,8191,0.0352960005402565
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,8191,0.025573333104451496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,8191,0.025290665527184803
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,8191,0.025989333788553875
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,8191,0.024618667860825855
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,8191,0.025477332373460133
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,8191,0.02553066611289978
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,8191,0.02370133250951767
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,8191,0.02364266663789749
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,8191,0.021221332252025604
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,8191,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,8191,0.01940800001223882
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,8191,0.019530666371186573
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,8191,0.019386666516462963
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,16383,0.04836800197760264
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,16383,0.05053866902987162
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,16383,0.05846933523813883
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,16383,0.04739200075467428
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,16383,0.04710933566093445
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,16383,0.048207998275756836
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,16383,0.04776533444722494
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,16383,0.04753066599369049
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,16383,0.031583999594052635
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,16383,0.03052799900372823
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,16383,0.03359466542800268
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,16383,0.025311999022960663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,16383,0.02404266595840454
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,16383,0.02536533276240031
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,16383,0.025722667574882507
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,16383,0.02348266790310542
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,32767,0.07315200070540111
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,32767,0.07833600044250488
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,32767,0.09668800234794617
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,32767,0.07449066638946533
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,32767,0.07341333230336507
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,32767,0.07399466633796692
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,32767,0.07287466526031494
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,32767,0.0711893339951833
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,32767,0.04971200227737427
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,32767,0.0489333321650823
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,32767,0.0524586687485377
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,32767,0.042208001017570496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,32767,0.0408693328499794
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,32767,0.04028266668319702
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,32767,0.041696002086003624
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,32767,0.041509332756201424
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,65535,0.11992533008257548
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,65535,0.1281599998474121
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,65535,0.17037866512934366
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,65535,0.1243946651617686
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,65535,0.11969066659609477
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,65535,0.12144000331560771
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,65535,0.11962133646011353
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,65535,0.12132799625396729
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,65535,0.0735999991496404
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,65535,0.07043200234572093
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,65535,0.08190399905045827
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,65535,0.06390400230884552
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,65535,0.06228266656398773
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,65535,0.0628053347269694
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,65535,0.062319998939832054
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,65535,0.06292800108591716
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,8,1,1,131071,0.21724265813827515
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,8,1,2,131071,0.23918932676315308
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,8,1,4,131071,0.33610665798187256
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,8,1,8,131071,0.23123733202616373
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,8,1,16,131071,0.22998400529225668
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,8,1,32,131071,0.22914665937423706
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,8,1,64,131071,0.228767991065979
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,8,1,128,131071,0.228218674659729
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,8,1,1,131071,0.12478933731714885
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,8,1,2,131071,0.11501333117485046
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,8,1,4,131071,0.14486400286356607
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,8,1,8,131071,0.1042133371035258
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,8,1,16,131071,0.10342933734258015
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,8,1,32,131071,0.10318932930628459
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,8,1,64,131071,0.10376532872517903
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,8,1,128,131071,0.10399466753005981
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,1,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,1,0.013733333597580591
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,1,0.013823999712864557
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,1,0.012703999876976013
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,1,0.01146666705608368
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,1,0.013610667238632837
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,1,0.011781333635250727
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,1,0.011328000575304031
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,1,0.01339200014869372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,1,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,1,0.01267733300725619
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,1,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,1,0.01126933346192042
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,1,0.012165332833925882
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,1,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,3,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,1,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,3,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,3,0.012970666090647379
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,3,0.012378666549921036
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,3,0.011338666081428528
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,3,0.01166933278242747
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,3,0.013343999783198038
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,3,0.011061333119869232
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,3,0.015578666081031164
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,3,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,3,0.012847999731699625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,3,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,3,0.011359999577204386
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,3,0.011744000017642975
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,3,0.011509332805871964
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,3,0.011567999919255575
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,7,0.015226667126019796
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,7,0.013765333841244379
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,7,0.01321600005030632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,7,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,7,0.012026666353146235
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,7,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,7,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,7,0.011450666934251785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,7,0.01471466695268949
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,7,0.01320533330241839
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,7,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,7,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,7,0.011205332974592844
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,7,0.01119999960064888
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,7,0.011402666568756104
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,15,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,7,0.011264000087976456
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,15,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,15,0.012831999609867731
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,15,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,15,0.012815999488035837
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,15,0.011066666493813196
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,15,0.011157333850860596
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,15,0.011237333218256632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,15,0.013770667215188345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,15,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,15,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,15,0.011909333368142446
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,15,0.012741333494583765
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,15,0.011018666128317514
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,15,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,15,0.011215999722480774
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,31,0.015552000453074774
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,31,0.014677333335081736
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,31,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,31,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,31,0.012661332885424295
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,31,0.011098666737476984
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,31,0.0138026662170887
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,31,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,31,0.013194666554530462
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,31,0.012890666723251343
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,31,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,31,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,31,0.012005332857370377
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,31,0.011285333583752314
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,31,0.01121066634853681
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,63,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,31,0.011578666667143503
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,63,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,63,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,63,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,63,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,63,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,63,0.011434666812419891
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,63,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,63,0.015061333775520325
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,63,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,63,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,63,0.011370666325092316
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,63,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,63,0.011194666226704916
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,63,0.01239466667175293
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,63,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,127,0.015568000574906668
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,127,0.013178666432698568
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,127,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,127,0.012773333738247553
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,127,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,127,0.01118933285276095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,127,0.011605333536863327
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,127,0.011173332730929056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,127,0.012858666479587555
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,127,0.013722666849692663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,127,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,127,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,127,0.011071999867757162
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,127,0.011519999553759893
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,127,0.011802667131026586
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,127,0.011557333171367645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,255,0.014848000059525171
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,255,0.013690666606028875
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,255,0.013712000101804733
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,255,0.012762666990359625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,255,0.011727999895811081
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,255,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,255,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,255,0.011503999431928
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,255,0.015578666081031164
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,255,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,255,0.01328533391157786
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,255,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,255,0.013568000247081121
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,255,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,255,0.011823999385039011
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,255,0.011690666278203329
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,511,0.020015999674797058
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,511,0.019029332945744198
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,511,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,511,0.014997333288192749
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,511,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,511,0.013882666826248169
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,511,0.013514666507641474
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,511,0.012991999586423239
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,511,0.015749332805474598
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,511,0.013610667238632837
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,511,0.015509333461523056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,511,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,511,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,511,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,511,0.013552000125249227
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,511,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,1023,0.02533866713444392
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,1023,0.021776000658671062
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,1023,0.01738133281469345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,1023,0.015573333948850632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,1023,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,1023,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,1023,0.014896000425020853
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,1023,0.015210667004187902
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,1023,0.01942933350801468
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,1023,0.017477333545684814
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,1023,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,1023,0.014325333138306936
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,1023,0.012768000364303589
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,1023,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,1023,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,2047,0.022175999979178112
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,1023,0.013898666948080063
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,2047,0.030000001192092896
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,2047,0.023391999304294586
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,2047,0.018816000471512478
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,2047,0.019626667102177937
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,2047,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,2047,0.01740266631046931
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,2047,0.01803733284274737
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,2047,0.02130666623512904
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,2047,0.02145066608985265
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,2047,0.017157333592573803
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,2047,0.017077332983414333
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,2047,0.015520000209410986
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,2047,0.015466666469971338
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,2047,0.01540800059835116
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,2047,0.015402667224407196
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,4095,0.027482666075229645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,4095,0.030389333764712017
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,4095,0.03165333221356074
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,4095,0.0244159996509552
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,4095,0.023344000180562336
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,4095,0.02342933416366577
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,4095,0.023813332120577495
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,4095,0.023386667172114056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,4095,0.0232640008131663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,4095,0.023621333142121632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,4095,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,4095,0.01918399954835574
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,4095,0.01913600042462349
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,4095,0.01844800015290578
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,4095,0.017370666066805523
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,4095,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,8191,0.04971200227737427
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,8191,0.051589335004488625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,8191,0.05605333546797434
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,8191,0.04699199895064036
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,8191,0.046725332736968994
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,8191,0.04710400104522705
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,8191,0.0451200008392334
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,8191,0.04623466730117798
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,8191,0.031557333966096245
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,8191,0.02959999938805898
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,8191,0.027722666660944622
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,8191,0.024170666933059692
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,8191,0.023434666295846302
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,8191,0.023002666731675465
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,8191,0.023647998770078022
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,8191,0.024005333582560223
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,16383,0.0758133331934611
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,16383,0.07821866869926453
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,16383,0.09540800253550212
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,16383,0.07542933523654938
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,16383,0.07354133327802022
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,16383,0.07327466706434886
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,16383,0.07390399773915608
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,16383,0.07306666672229767
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,16383,0.05023466547330221
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,16383,0.048325334986050926
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,16383,0.050698667764663696
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,16383,0.039887999494870506
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,16383,0.04081066697835922
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,16383,0.03976533313592275
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,16383,0.03828266759713491
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,16383,0.04055466751257578
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,32767,0.12819733222325644
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,32767,0.1269599994023641
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,32767,0.17810134092966715
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,32767,0.12311466534932454
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,32767,0.12385599811871846
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,32767,0.1222826639811198
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,32767,0.12076800068219502
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,32767,0.121888001759847
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,32767,0.07747200131416321
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,32767,0.07201600074768066
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,32767,0.08604266246159871
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,32767,0.06460799773534139
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,32767,0.06358933448791504
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,32767,0.06364800035953522
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,32767,0.0625439981619517
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,32767,0.06288533409436543
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,65535,0.22776534159978232
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,65535,0.22919466098149618
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,65535,0.32839999596277875
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,65535,0.22336000204086304
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,65535,0.2226453423500061
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,65535,0.2222613294919332
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,65535,0.22073066234588623
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,65535,0.22235200802485147
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,65535,0.1330400009950002
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,65535,0.1143946647644043
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,65535,0.15381866693496704
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,65535,0.10429333647092183
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,65535,0.1037493348121643
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,65535,0.10346133510271709
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,65535,0.10337600111961365
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,65535,0.10355732838312785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,16,1,1,131071,0.4296319882074992
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,16,1,2,131071,0.4349386692047119
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,16,1,4,131071,0.6498399972915649
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,16,1,8,131071,0.423093318939209
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,16,1,16,131071,0.4157493511835734
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,16,1,32,131071,0.4166773160298665
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,16,1,64,131071,0.41793068250020343
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,16,1,128,131071,0.41970666249593097
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,16,1,1,131071,0.2420533299446106
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,16,1,2,131071,0.2044853369394938
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,16,1,4,131071,0.2906399965286255
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,16,1,8,131071,0.18721065918604532
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,16,1,16,131071,0.18646933635075888
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,16,1,32,131071,0.18703466653823853
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,1,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,16,1,64,131071,0.18635733922322592
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,1,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,16,1,128,131071,0.18574400742848715
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,1,0.01444799949725469
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,1,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,1,0.013066666821638743
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,1,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,1,0.013359999905029932
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,1,0.01693333312869072
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,1,0.012725333372751871
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,1,0.013114667187134424
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,1,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,1,0.011120000233252844
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,1,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,1,0.012634667257467905
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,1,0.013072000195582708
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,1,0.01110400011142095
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,3,0.014991999914248785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,3,0.015626666446526844
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,3,0.013514666507641474
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,3,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,3,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,3,0.013365333278973898
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,3,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,3,0.012944000462690989
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,3,0.014794666320085526
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,3,0.01563199982047081
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,3,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,3,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,3,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,3,0.011258666714032492
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,3,0.012266666938861212
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,7,0.015002666662136713
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,3,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,7,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,7,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,7,0.013232000172138214
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,7,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,7,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,7,0.013173333058754602
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,7,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,7,0.017114666601022083
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,7,0.014789332946141561
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,7,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,7,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,7,0.01312000056107839
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,7,0.012778667112191519
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,7,0.012837332983811697
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,7,0.013381333400805792
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,15,0.015477333217859268
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,15,0.015589332828919092
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,15,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,15,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,15,0.013647999614477158
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,15,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,15,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,15,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,15,0.016773333152135212
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,15,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,15,0.013818666338920593
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,15,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,15,0.01181866725285848
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,15,0.011322667201360067
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,15,0.011242666592200598
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,15,0.011855999628702799
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,31,0.014805333067973455
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,31,0.015599999576807022
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,31,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,31,0.0129120002190272
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,31,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,31,0.012784000486135483
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,31,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,31,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,31,0.01666133354107539
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,31,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,31,0.013471999516089758
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,31,0.01292266696691513
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,31,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,31,0.011168000598748526
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,31,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,31,0.011183999478816986
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,63,0.015520000209410986
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,63,0.015098666151364645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,63,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,63,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,63,0.013487999637921652
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,63,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,63,0.01309866706530253
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,63,0.01303999995191892
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,63,0.015381333728631338
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,63,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,63,0.015018666783968607
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,63,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,63,0.011002667248249054
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,63,0.012842666357755661
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,63,0.012624000509579977
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,63,0.013002666334311167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,127,0.015386667102575302
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,127,0.01544533297419548
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,127,0.013562666873137156
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,127,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,127,0.013594667116800943
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,127,0.012810666114091873
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,127,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,127,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,127,0.01647466669480006
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,127,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,127,0.013082666943470636
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,127,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,127,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,127,0.011141333729028702
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,127,0.011125333607196808
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,127,0.012863999853531519
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,255,0.01481066644191742
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,255,0.021429332594076794
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,255,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,255,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,255,0.013546666751305262
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,255,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,255,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,255,0.012853333105643591
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,255,0.014677333335081736
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,255,0.0229066660006841
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,255,0.012682666381200155
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,255,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,255,0.012650666137536367
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,255,0.013151999562978745
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,255,0.011477333803971609
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,255,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,511,0.02180800090233485
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,511,0.020069333414236706
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,511,0.017221332838137943
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,511,0.015498666713635126
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,511,0.013877333452304205
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,511,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,511,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,511,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,511,0.022410665949185688
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,511,0.01658133293191592
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,511,0.01534933348496755
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,511,0.019744000087181728
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,511,0.012805332740147909
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,511,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,511,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,511,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,1023,0.02382933348417282
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,1023,0.02550400048494339
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,1023,0.02182399978240331
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,1023,0.01759999990463257
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,1023,0.01703466723362605
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,1023,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,1023,0.017418666432301205
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,1023,0.01720000058412552
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,1023,0.025455998877684276
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,1023,0.01929066702723503
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,1023,0.017525333911180496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,1023,0.015253332753976187
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,1023,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,1023,0.013503999759753546
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,1023,0.014741333822409311
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,1023,0.01421333352724711
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,2047,0.031471999982992806
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,2047,0.028175999720891316
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,2047,0.03195200115442276
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,2047,0.02160000056028366
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,2047,0.02207999924818675
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,2047,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,2047,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,2047,0.02146133283774058
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,2047,0.027989332874615986
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,2047,0.02346133440732956
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,2047,0.021344001094500225
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,2047,0.01764800027012825
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,2047,0.01695999999841054
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,2047,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,2047,0.015487999965747198
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,2047,0.01637866720557213
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,4095,0.050479998191197716
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,4095,0.05212800204753876
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,4095,0.057328000664711
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,4095,0.04537599782148997
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,4095,0.04191466669241587
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,4095,0.04383466641108195
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,4095,0.0447626660267512
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,4095,0.04389866689840952
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,4095,0.033786666889985405
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,4095,0.02762666592995326
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,4095,0.03123733401298523
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,4095,0.021568000316619873
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,4095,0.02109333376089732
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,4095,0.021568000316619873
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,4095,0.021007999777793884
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,4095,0.01974933346112569
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,8191,0.07789333164691925
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,8191,0.0788373351097107
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,8191,0.09730666875839233
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,8191,0.07382933298746745
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,8191,0.07132799923419952
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,8191,0.07211733361085255
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,8191,0.07137600084145863
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,8191,0.0711359977722168
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,8191,0.053130666414896645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,8191,0.04829333225886027
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,8191,0.051738664507865906
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,8191,0.037845333417256675
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,8191,0.03679466744263967
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,8191,0.03700799991687139
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,8191,0.0379573330283165
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,8191,0.03648533423741659
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,16383,0.12844266494115195
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,16383,0.13038933277130127
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,16383,0.17308266957600912
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,16383,0.12738133470217386
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,16383,0.12204266587893169
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,16383,0.12447999914487202
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,16383,0.1239520013332367
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,16383,0.12377599875132243
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,16383,0.08031466603279114
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,16383,0.07158400118350983
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,16383,0.08506133159001668
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,16383,0.06047999858856201
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,16383,0.06007466713587443
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,16383,0.059077332417170204
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,16383,0.05936000247796377
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,16383,0.061039999127388
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,32767,0.22872000932693481
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,32767,0.23219199975331625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,32767,0.32890133062998456
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,32767,0.22979199886322021
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,32767,0.22816000382105509
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,32767,0.23058666785558066
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,32767,0.22845866282780966
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,32767,0.23094399770100912
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,32767,0.11877333124478658
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,32767,0.1349386672178904
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,32767,0.15555733442306519
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,32767,0.10584533214569092
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,32767,0.10363733768463135
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,32767,0.10359467069307964
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,32767,0.10481066505114238
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,32767,0.10430933038393657
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,65535,0.43346667289733887
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,65535,0.4381493330001831
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,65535,0.6571786801020304
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,65535,0.4411040147145589
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,65535,0.43585066000620526
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,65535,0.43159464995066327
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,65535,0.4355786641438802
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,65535,0.43411731719970703
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,65535,0.2433333396911621
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,65535,0.21282132466634116
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,65535,0.292138675848643
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,65535,0.19107200702031454
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,65535,0.18981333573659262
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,65535,0.19097065925598145
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,65535,0.1893600026766459
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,65535,0.190720001856486
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,32,1,1,131071,0.8372266292572021
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,32,1,2,131071,0.8525546391805013
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,32,1,8,131071,0.850762685139974
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,32,1,4,131071,1.3292746543884277
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,32,1,16,131071,0.840773344039917
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,32,1,32,131071,0.8399306933085123
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,32,1,64,131071,0.8375253677368164
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,32,1,128,131071,0.8374506632486979
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,32,1,1,131071,0.4615253210067749
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,32,1,2,131071,0.40075735251108807
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,32,1,4,131071,0.5675626595815023
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,32,1,8,131071,0.3671199878056844
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,32,1,16,131071,0.3661919832229614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,1,0.01532799998919169
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,32,1,32,131071,0.365882674853007
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,32,1,128,131071,0.36486931641896564
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,32,1,64,131071,0.3641706705093384
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,1,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,1,0.015354666858911514
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,1,0.012938667088747025
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,1,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,1,0.0136266661187013
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,1,0.013386666774749756
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,1,0.012800000607967377
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,1,0.019007999449968338
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,1,0.015263999501864115
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,1,0.013872000078360239
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,1,0.012789333860079447
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,1,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,1,0.012730666746695837
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,1,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,1,0.011274666835864386
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,3,0.016202667107184727
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,3,0.01488000030318896
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,3,0.015583999454975128
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,3,0.013647999614477158
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,3,0.013306666165590286
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,3,0.013397333522637686
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,3,0.013829333086808523
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,3,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,3,0.017514667163292568
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,3,0.014938666174809137
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,3,0.015381333728631338
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,3,0.012901333471139273
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,3,0.013167999684810638
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,3,0.013338666409254074
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,3,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,3,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,7,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,7,0.014943999548753103
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,7,0.014901333798964819
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,7,0.013983999689420065
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,7,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,7,0.013765333841244379
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,7,0.014277332772811254
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,7,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,7,0.01728533332546552
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,7,0.014789332946141561
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,7,0.013893333574136099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,7,0.013663999736309052
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,7,0.013557333499193192
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,7,0.012650666137536367
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,7,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,7,0.012997332960367203
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,15,0.015376000354687372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,15,0.015344000111023584
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,15,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,15,0.01421333352724711
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,15,0.013786666095256805
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,15,0.013237333546082178
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,15,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,15,0.013482666263977686
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,15,0.019296000401178997
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,15,0.015573333948850632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,15,0.013616000612576803
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,15,0.012949333836634954
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,15,0.013269333789745966
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,15,0.013056000073750814
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,15,0.01137599969903628
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,15,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,31,0.016037333756685257
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,31,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,31,0.015423999478419622
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,31,0.01451733335852623
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,31,0.013525333255529404
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,31,0.013861333330472311
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,31,0.01314666618903478
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,31,0.013141332815090815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,31,0.019359999646743137
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,31,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,31,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,31,0.013605333864688873
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,31,0.012757333616415659
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,31,0.013264000415802002
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,31,0.013093333691358566
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,31,0.013077333569526672
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,63,0.015637333194414776
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,63,0.01570133368174235
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,63,0.015082667271296183
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,63,0.013253333667914072
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,63,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,63,0.01370666672786077
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,63,0.013909333695967993
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,63,0.013568000247081121
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,63,0.01749333366751671
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,63,0.015754666179418564
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,63,0.014384000251690546
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,63,0.013050666699806849
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,63,0.013274667163689932
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,63,0.013013333082199097
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,63,0.011413333316644033
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,63,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,127,0.015306666493415833
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,127,0.013370666652917862
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,127,0.015040000279744467
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,127,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,127,0.015034666905800501
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,127,0.013493333011865616
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,127,0.01313599944114685
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,127,0.013637332866589228
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,127,0.01915733392039935
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,127,0.015285332997639975
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,127,0.013440000514189402
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,127,0.013376000026861826
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,127,0.012986666212479273
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,127,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,127,0.011535999675591787
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,127,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,255,0.01735466718673706
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,255,0.021946666141351063
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,255,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,255,0.013562666873137156
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,255,0.013007999708255133
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,255,0.013552000125249227
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,255,0.013354666531085968
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,255,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,255,0.019013332823912304
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,255,0.022730665902296703
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,255,0.013850666582584381
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,255,0.014069333672523499
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,255,0.013189333180586496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,255,0.013157332936922709
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,255,0.012981332838535309
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,255,0.012533333152532578
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,511,0.019440000255902607
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,511,0.02365333338578542
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,511,0.019194666296243668
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,511,0.017952000101407368
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,511,0.015365333606799444
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,511,0.01701333373785019
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,511,0.01706133286158244
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,511,0.01573866605758667
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,511,0.02319466571013133
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,511,0.020069333414236706
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,511,0.015184000134468079
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,511,0.015402667224407196
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,511,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,511,0.01434133326013883
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,511,0.01341333364446958
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,511,0.01505600040157636
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,1023,0.025727999707063038
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,1023,0.031386665999889374
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,1023,0.029701332251230877
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,1023,0.02306666721900304
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,1023,0.020997333029905956
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,1023,0.02128533273935318
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,1023,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,1023,0.02145066608985265
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,1023,0.022970666488011677
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,1023,0.025701334079106648
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,1023,0.019098666807015736
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,1023,0.01747200017174085
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,1023,0.015317333241303762
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,1023,0.015200000256299973
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,1023,0.015050667027632395
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,1023,0.01553600033124288
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,2047,0.04599999884764353
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,2047,0.05300266544024149
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,2047,0.05376000205675761
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,2047,0.04970666766166687
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,2047,0.04236799975236257
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,2047,0.044719999035199486
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,2047,0.04571199913819631
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,2047,0.04524800181388855
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,2047,0.030378667016824085
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,2047,0.031136001149813335
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,2047,0.02844800055027008
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,2047,0.02085866779088974
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,2047,0.019461333751678467
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,2047,0.01930133377512296
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,2047,0.019296000401178997
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,2047,0.019391999890406925
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,4095,0.07100800176461537
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,4095,0.08149866759777069
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,4095,0.09185066819190979
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,4095,0.07528533538182576
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,4095,0.07196799914042155
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,4095,0.07216533521811168
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,4095,0.0710453341404597
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,4095,0.07237333556016286
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,4095,0.047968000173568726
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,4095,0.05057600140571594
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,4095,0.050474668542544045
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,4095,0.03978666663169861
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,4095,0.037733333806196846
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,4095,0.03751466671625773
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,4095,0.03826133410135905
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,4095,0.03705599904060364
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,8191,0.12345066666603088
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,8191,0.1327946682771047
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,8191,0.16611733039220175
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,8191,0.12752532958984375
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,8191,0.12434132893880208
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,8191,0.12309333682060242
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,8191,0.12335999806722005
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,8191,0.12292800347010295
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,8191,0.07297599812348683
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,8191,0.07456533114115398
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,8191,0.0620959997177124
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,8191,0.08339200417200725
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,8191,0.05898133416970571
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,8191,0.05991999804973602
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,8191,0.058517331878344216
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,8191,0.05951466659704844
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,16383,0.22503467400868735
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,16383,0.23425600926081339
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,16383,0.3148639996846517
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,16383,0.23043733835220337
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,16383,0.23107733329137167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,16383,0.23033599058787027
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,16383,0.22829333941141763
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,16383,0.22827200094858804
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,16383,0.12492266297340393
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,16383,0.12261866529782613
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,16383,0.15340266625086466
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,16383,0.10572800040245056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,16383,0.1060693363348643
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,16383,0.10445333520571391
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,16383,0.10450667142868042
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,16383,0.10462400317192078
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,32767,0.42981334527333576
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,32767,0.43794135252634686
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,32767,0.6135253508885702
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,32767,0.44301335016886395
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,32767,0.4365813334782918
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,32767,0.4359999895095825
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,32767,0.43290666739145917
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,32767,0.43565865357716876
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,32767,0.2239733338356018
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,32767,0.21575466791788736
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,32767,0.2872106631596883
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,32767,0.19301333030064902
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,32767,0.19160000483194986
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,32767,0.19005332390467325
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,32767,0.1913599967956543
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,32767,0.18969599405924478
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,65535,0.8394026756286621
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,65535,0.8529173533121744
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,65535,1.2176693280537922
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,65535,0.8468373616536459
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,65535,0.8417332967122396
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,65535,0.8429439862569174
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,65535,0.8381546338399252
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,65535,0.8390773137410482
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,65535,0.4225120147069295
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,65535,0.40332265694936115
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,65535,0.5646666685740153
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,65535,0.36798401673634845
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,65535,0.3667360146840413
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,65535,0.36558934052785236
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,65535,0.3652160167694092
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,65535,0.3657653331756592
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,64,1,1,131071,1.6582345962524414
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,64,1,2,131071,1.6692907015482585
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,64,1,8,131071,1.663690725962321
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,64,1,4,131071,2.4136106173197427
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,64,1,16,131071,1.6379680633544922
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,64,1,32,131071,1.636975924173991
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,64,1,64,131071,1.6401546796162922
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,64,1,128,131071,1.6407787005106609
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,64,1,1,131071,0.8210773468017578
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,64,1,2,131071,0.7782080173492432
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,64,1,4,131071,1.107413371404012
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,64,1,8,131071,0.7172373135884603
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,64,1,16,131071,0.71506667137146
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,64,1,32,131071,0.7151306470235189
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,64,1,64,131071,0.7139200369517008
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,1,0.01974933346112569
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,1,0.017621333400408428
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,1,0.021525333325068157
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,1,0.015130666395028433
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,1,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,1,0.01526933287580808
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,64,1,128,131071,0.7135787010192871
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,1,0.014671999961137772
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,1,0.014826666563749313
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,1,0.025674665967623394
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,1,0.02235200007756551
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,1,0.019744000087181728
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,1,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,1,0.013248000293970108
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,1,0.013210666676362356
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,1,0.013221333424250284
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,1,0.014453332871198654
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,3,0.019237333287795384
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,3,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,3,0.015450666348139444
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,3,0.020768000433842342
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,3,0.015205333630243937
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,3,0.014511999984582266
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,3,0.015301333119471868
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,3,0.015066667149464289
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,3,0.02534399926662445
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,3,0.017557332913080852
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,3,0.018746666610240936
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,3,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,3,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,3,0.012869333227475485
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,3,0.01332266628742218
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,3,0.013104000439246496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,7,0.021157334248224895
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,7,0.0164533331990242
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,7,0.015397333850463232
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,7,0.021482666333516438
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,7,0.014560000350077948
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,7,0.015013333410024643
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,7,0.015103999525308609
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,7,0.015173333386580149
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,7,0.025589334468046825
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,7,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,7,0.013925333817799887
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,7,0.01972266659140587
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,7,0.01322666679819425
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,7,0.013130666067202887
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,7,0.013034666577974955
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,15,0.01926400015751521
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,7,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,15,0.016389333953460056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,15,0.021317332983016968
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,15,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,15,0.015125333021084467
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,15,0.015322666615247726
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,15,0.01504533365368843
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,15,0.01498666654030482
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,15,0.025663999219735462
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,15,0.01942933350801468
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,15,0.014186666657527288
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,15,0.019082666685183842
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,15,0.012965332716703415
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,15,0.01331199953953425
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,15,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,31,0.021216000119845074
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,15,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,31,0.017829333742459614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,31,0.02067199970285098
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,31,0.015370666980743408
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,31,0.015178666760524115
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,31,0.014352000008026758
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,31,0.015461333096027374
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,31,0.015146666516860327
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,31,0.025450666745503742
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,31,0.01722666621208191
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,31,0.01961600035429001
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,31,0.015114666273196539
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,31,0.01350933313369751
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,31,0.013461332768201828
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,31,0.013663999736309052
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,31,0.013088000317414602
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,63,0.02144533395767212
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,63,0.01684800038735072
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,63,0.021338666478792827
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,63,0.015216000378131866
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,63,0.015311999867359797
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,63,0.015194666882356008
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,63,0.013786666095256805
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,63,0.014629332969586054
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,63,0.025936000049114227
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,63,0.019199999670187633
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,63,0.019130667050679524
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,63,0.013487999637921652
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,63,0.013045333325862885
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,63,0.013450667262077332
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,63,0.01302933320403099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,63,0.013199999928474426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,127,0.01699200024207433
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,127,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,127,0.0200853335360686
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,127,0.01522133375207583
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,127,0.015168000012636185
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,127,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,127,0.01516266663869222
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,127,0.015072000523408255
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,127,0.02593066543340683
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,127,0.018901333212852478
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,127,0.01959466685851415
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,127,0.013530666629473368
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,127,0.012975999464591345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,127,0.013573333621025085
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,127,0.012917333592971167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,127,0.013807999591032663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,255,0.02537599951028824
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,255,0.01729600007335345
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,255,0.02163200080394745
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,255,0.015781333049138386
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,255,0.013242666920026144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,255,0.015493333339691162
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,255,0.015541333705186844
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,255,0.013162666310866674
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,255,0.02789866675933202
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,255,0.019002666076024372
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,255,0.019551999866962433
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,255,0.013183999806642532
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,255,0.013445333888133367
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,255,0.013258667041858038
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,255,0.013722666849692663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,255,0.01333333303531011
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,511,0.032170665760835014
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,511,0.025231999655564625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,511,0.032746667663256325
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,511,0.019472000499566395
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,511,0.018629333625237148
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,511,0.018101333330074947
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,511,0.018138666947682697
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,511,0.017322666943073273
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,511,0.0317493329445521
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,511,0.022976001103719074
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,511,0.021210665504137676
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,511,0.015589332828919092
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,511,0.014965333044528961
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,511,0.01543466622630755
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,511,0.015141333142916361
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,511,0.01545599972208341
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,1023,0.04983466863632202
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,1023,0.04874666531880697
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,1023,0.05384000142415365
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,1023,0.04408533374468485
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,1023,0.04219200213750204
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,1023,0.04051200052102407
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,1023,0.04201599955558777
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,1023,0.042064001162846885
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,1023,0.04196266829967499
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,1023,0.027722666660944622
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,1023,0.03369600077470144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,1023,0.019258666783571243
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,1023,0.01931200052301089
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,1023,0.01829333355029424
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,1023,0.018496000518401463
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,1023,0.019167999426523846
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,2047,0.0769760012626648
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,2047,0.07481599847475688
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,2047,0.07096000015735626
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,2047,0.09568533301353455
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,2047,0.069882666071256
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,2047,0.06692266464233398
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,2047,0.06817600131034851
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,2047,0.06853333115577698
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,2047,0.05648533503214518
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,2047,0.04769066472848257
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,2047,0.053818667928377785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,2047,0.03811733424663544
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,2047,0.03589333345492681
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,2047,0.037104000647862755
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,2047,0.03633599976698557
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,2047,0.03675200045108795
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,4095,0.12828800082206726
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,4095,0.12797333796819052
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,4095,0.122789333264033
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,4095,0.16921599706013998
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,4095,0.12237333257993062
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,4095,0.11999467015266418
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,4095,0.11948800086975098
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,4095,0.12015466888745625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,4095,0.08283199866612752
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,4095,0.0710346649090449
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,4095,0.08833066622416179
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,4095,0.060602664947509766
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,4095,0.05905066430568695
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,4095,0.058549334605534874
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,4095,0.05824000140031179
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,4095,0.05896000067392985
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,8191,0.23295466105143228
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,8191,0.2288853327433268
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,8191,0.31894399722417194
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,8191,0.22390933831532797
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,8191,0.22294400135676065
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,8191,0.22075732549031576
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,8191,0.22286399205525717
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,8191,0.22405866781870523
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,8191,0.13539200027783713
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,8191,0.11788800358772278
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,8191,0.15730133652687073
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,8191,0.10534399747848511
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,8191,0.10538132985432942
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,8191,0.10453333457310994
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,8191,0.10358933607737224
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,8191,0.10454400380452473
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,16383,0.4369493325551351
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,16383,0.43340798219045
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,16383,0.6186079978942871
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,16383,0.42601601282755536
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,16383,0.4240800142288208
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,16383,0.426581343015035
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,16383,0.4257813294728597
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,16383,0.42601064840952557
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,16383,0.2370026707649231
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,16383,0.21277866760889688
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,16383,0.2930613358815511
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,16383,0.19195199012756348
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,16383,0.1903839906056722
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,16383,0.1906986633936564
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,16383,0.19035200277964273
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,16383,0.19003732999165854
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,32767,0.8486239910125732
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,32767,0.8496320247650146
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,32767,1.2243893146514893
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,32767,0.8242560227711996
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,32767,0.8235519727071127
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,32767,0.8260533014933268
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,32767,0.8215040365854899
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,32767,0.832757314046224
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,32767,0.43581334749857586
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,32767,0.4006719986597697
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,32767,0.36531734466552734
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,32767,0.5669920047124227
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,32767,0.36456533273061115
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,32767,0.3646453221638997
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,32767,0.36302932103474933
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,32767,0.36423468589782715
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,65535,1.67302401860555
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,65535,1.660794734954834
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,65535,1.6140906016031902
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,65535,2.5422239303588867
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,65535,1.6174720128377278
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,65535,1.6145706176757812
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,65535,1.6167999903361003
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,65535,1.6084853808085124
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,65535,0.8380800088246664
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,65535,0.7759466965993246
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,65535,0.7144959767659506
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,65535,1.11626132329305
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,65535,0.7134400208791097
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,65535,0.7141226927439371
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,65535,0.7111732959747314
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,65535,0.7114240328470866
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,128,1,1,131071,3.318976084391276
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,128,1,2,131071,3.379066785176595
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,128,1,8,131071,3.257237434387207
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,128,1,4,131071,5.062586784362793
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,128,1,16,131071,3.25054931640625
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,128,1,32,131071,3.2504746119181314
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,128,1,64,131071,3.2527360916137695
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,128,1,128,131071,3.2423038482666016
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,128,1,1,131071,1.6443573633829753
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,128,1,2,131071,1.5265013376871746
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,128,1,4,131071,2.242565313975016
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,128,1,8,131071,1.4130239486694336
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,128,1,16,131071,1.409429391225179
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,1,0.03150933235883713
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,128,1,32,131071,1.4114400545756023
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,1,0.023045333723227184
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,1,0.031301334500312805
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,1,0.021173333128293354
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,128,1,64,131071,1.4082454045613606
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,1,0.020501332978407543
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,128,1,128,131071,1.4104906717936199
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,1,0.02073066681623459
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,1,0.019882666567961376
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,1,0.01969066634774208
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,1,0.044362664222717285
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,1,0.02664000044266383
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,1,0.02703999976317088
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,1,0.019472000499566395
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,1,0.016704000532627106
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,1,0.01770666614174843
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,1,0.0161920003592968
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,1,0.015706667055686314
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,3,0.030581332743167877
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,3,0.02327466756105423
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,3,0.02958400050799052
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,3,0.021141332884629566
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,3,0.020047999918460846
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,3,0.01921066641807556
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,3,0.02146133283774058
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,3,0.01947733387351036
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,3,0.04256533086299896
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,3,0.026352000733216602
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,3,0.027221334477265675
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,3,0.019413333386182785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,3,0.017231999586025875
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,3,0.01730666682124138
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,3,0.017914666483799618
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,3,0.016949333250522614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,7,0.03183999905983607
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,7,0.023423999547958374
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,7,0.031557333966096245
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,7,0.02179733415444692
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,7,0.020314666132132213
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,7,0.019434666881958645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,7,0.019925333559513092
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,7,0.04359999795754751
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,7,0.02254933367172877
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,7,0.02585600068171819
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,7,0.027509334186712902
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,7,0.01923199991385142
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,7,0.017504000415404636
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,7,0.016783999900023144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,7,0.017509333789348602
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,7,0.0161013330022494
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,15,0.030405332644780476
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,15,0.023103999594847362
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,15,0.029717333614826202
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,15,0.021018666525681812
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,15,0.02004266654451688
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,15,0.019861333072185516
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,15,0.02110933264096578
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,15,0.019754666835069656
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,15,0.043525333205858864
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,15,0.02658133457104365
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,15,0.027461332579453785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,15,0.016965333372354507
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,15,0.01886933296918869
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,15,0.015658666690190632
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,15,0.017103999853134155
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,15,0.01692266638080279
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,31,0.03160000095764796
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,31,0.021557333568731945
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,31,0.03133333226044973
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,31,0.021781332790851593
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,31,0.02075733368595441
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,31,0.019621333728233974
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,31,0.020629333953062694
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,31,0.019808000574509304
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,31,0.04390933116277059
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,31,0.025349333882331848
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,31,0.027503999571005504
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,31,0.019189332922299702
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,31,0.017797333498795826
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,31,0.015295999745527903
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,31,0.01741333305835724
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,63,0.03151999910672506
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,31,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,63,0.021312000850836437
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,63,0.03178666780392329
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,63,0.021557333568731945
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,63,0.019904000063737232
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,63,0.020954666038354237
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,63,0.019498666127522785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,63,0.019834666202465694
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,63,0.04394133388996124
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,63,0.027434666951497395
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,63,0.02646933247645696
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,63,0.019306667149066925
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,63,0.0169813334941864
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,63,0.016330666840076447
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,63,0.017162666966517765
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,63,0.01773333301146825
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,127,0.033674667278925575
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,127,0.0216799999276797
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,127,0.029722665747006733
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,127,0.021850667893886566
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,127,0.02029866725206375
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,127,0.02120000123977661
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,127,0.019648000597953796
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,127,0.021045332153638203
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,127,0.04413333535194397
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,127,0.027424000203609467
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,127,0.027610667049884796
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,127,0.01947733387351036
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,127,0.017237332959969837
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,127,0.01714133347074191
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,127,0.015498666713635126
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,127,0.017136000096797943
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,255,0.04429866870244344
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,255,0.030554667115211487
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,255,0.033413333197434746
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,255,0.02186133215824763
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,255,0.02011200040578842
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,255,0.021002667645613354
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,255,0.01937599976857503
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,255,0.020687999824682873
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,255,0.04997866849104563
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,255,0.029472000896930695
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,255,0.027461332579453785
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,255,0.01825599993268649
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,255,0.016693333784739178
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,255,0.015647999942302704
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,255,0.015605332950750986
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,255,0.015290666371583939
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,511,0.0577706644932429
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,511,0.05198933184146881
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,511,0.05726933479309082
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,511,0.04387733340263367
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,511,0.041989331444104515
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,511,0.04256533086299896
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,511,0.04250133534272512
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,511,0.042208001017570496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,511,0.05796800057093302
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,511,0.039994666973749794
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,511,0.03751999884843826
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,511,0.023893333971500397
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,511,0.02057066683967908
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,511,0.019871999820073444
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,511,0.021253332495689392
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,511,0.020501332978407543
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,1023,0.08516800403594971
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,1023,0.07904533545176189
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,1023,0.09629866480827332
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,1023,0.07148799796899159
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,1023,0.06938666601975758
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,1023,0.06971733272075653
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,1023,0.06654933094978333
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,1023,0.0680159976085027
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,1023,0.07293333113193512
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,1023,0.05429866909980774
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,1023,0.05619200070699056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,1023,0.04155733436346054
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,1023,0.03818133225043615
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,1023,0.03804266701141993
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,1023,0.03782399992148081
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,1023,0.037130666275819145
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,2047,0.14040533701578775
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,2047,0.1325866679350535
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,2047,0.17371733983357748
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,2047,0.12106666962305705
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,2047,0.11994133392969768
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,2047,0.12091199556986491
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,2047,0.11778666575749715
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,2047,0.11892267068227132
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,2047,0.09925333658854167
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,2047,0.08072533210118611
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,2047,0.09346666932106018
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,2047,0.06132266422112783
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,2047,0.07173333565394084
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,2047,0.06200533111890157
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,2047,0.060720001657803856
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,2047,0.0603413333495458
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,4095,0.2423306703567505
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,4095,0.23727466662724814
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,4095,0.31989866495132446
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,4095,0.22765332460403442
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,4095,0.2219466765721639
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,4095,0.22353067000706991
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,4095,0.223578671614329
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,4095,0.2236106594403585
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,4095,0.1527839998404185
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,4095,0.12963733077049255
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,4095,0.1623253325621287
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,4095,0.10672533512115479
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,4095,0.10992532968521118
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,4095,0.106495996316274
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,4095,0.10511466860771179
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,4095,0.10411733388900757
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,8191,0.4504479964574178
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,8191,0.4467093149820964
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,8191,0.6202239990234375
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,8191,0.4282613197962443
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,8191,0.43033067385355633
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,8191,0.42924801508585614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,8191,0.4275466601053874
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,8191,0.4256480137507121
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,8191,0.256933331489563
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,8191,0.29715200265248615
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,8191,0.22544533014297485
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,8191,0.19732799132665
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,8191,0.19581866264343262
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,8191,0.1943626602490743
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,8191,0.1937119960784912
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,8191,0.19320533672968546
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,16383,0.8605333169301351
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,16383,0.8604479630788168
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,16383,0.833130677541097
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,16383,1.2349759737650554
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,16383,0.8351519902547201
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,16383,0.8275573253631592
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,16383,0.8285813331604004
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,16383,0.8305493195851644
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,16383,0.45818134148915607
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,16383,0.4142133394877116
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,16383,0.5686773459116617
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,16383,0.3730986515680949
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,16383,0.36842668056488037
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,16383,0.3689866860707601
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,16383,0.3677866856257121
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,16383,0.3684320052464803
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,32767,1.6880693435668945
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,32767,1.7092533111572266
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,32767,2.4929280281066895
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,32767,1.6519625981648762
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,32767,1.649461269378662
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,32767,1.6553120613098145
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,32767,1.6529173851013184
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,32767,1.6568373044331868
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,32767,0.8634453614552816
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,32767,0.7919306755065918
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,32767,1.1164373556772869
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,32767,0.7227679888407389
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,32767,0.7203786373138428
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,32767,0.7210293610890707
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,32767,0.7208800315856934
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,32767,0.7175146738688151
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,256,1,1,65535,3.3431199391682944
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,256,1,2,65535,3.357968012491862
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,256,1,4,65535,5.127274513244629
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,256,1,8,65535,3.256154696146647
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,256,1,16,65535,3.251002629597982
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,256,1,32,65535,3.2559305826822915
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,256,1,64,65535,3.2512534459431968
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,256,1,128,65535,3.2487414677937827
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,256,1,1,65535,1.6731093724568684
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,256,1,2,65535,1.549546718597412
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,256,1,8,65535,1.4282453854878743
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,256,1,4,65535,2.2514826456705728
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,256,1,32,65535,1.4237386385599773
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,256,1,16,65535,1.4245492617289226
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,1,0.05189333359400431
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,1,0.036362667878468834
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,1,0.048122664292653404
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,1,0.031445334355036415
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,256,1,64,65535,1.4202720324198406
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,1,0.02902399996916453
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,256,1,128,65535,1.4229119618733723
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,1,0.029930666089057922
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,1,0.029178666571776073
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,1,0.03033600002527237
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,1,0.0790773332118988
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,1,0.04696000119050344
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,1,0.04363200068473816
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,1,0.026122666895389557
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,1,0.023370665808518726
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,1,0.022629333039124806
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,1,0.0235359991590182
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,1,0.021216000119845074
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,3,0.05297600229581197
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,3,0.0359253336985906
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,3,0.04799466828505198
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,3,0.031311998764673867
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,3,0.029834667841593426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,3,0.029466666281223297
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,3,0.029253333806991577
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,3,0.02977066735426585
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,3,0.07948266466458638
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,3,0.041984001795450844
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,3,0.046767999728520714
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,3,0.026447998980681103
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,3,0.023647998770078022
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,3,0.021984001000722248
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,3,0.023290666441122692
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,3,0.021477334201335907
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,7,0.05253333350022634
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,7,0.03583466758330663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,7,0.048197334011395775
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,7,0.030837332208951313
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,7,0.029472000896930695
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,7,0.0295413335164388
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,7,0.029493334392706554
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,7,0.030080000559488933
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,7,0.07952000200748444
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,7,0.0459199994802475
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,7,0.04460266729195913
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,7,0.02743999908367793
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,7,0.022122666239738464
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,7,0.023599999646345775
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,7,0.023455999791622162
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,7,0.02181333303451538
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,15,0.03570666660865148
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,15,0.05283733208974203
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,15,0.04845866560935974
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,15,0.032138665517171226
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,15,0.02941333254178365
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,15,0.030202666918436687
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,15,0.02939733366171519
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,15,0.029338667790095013
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,15,0.07893333335717519
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,15,0.04609066744645437
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,15,0.027280000348885853
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,15,0.0418453315893809
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,15,0.023658665517965954
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,15,0.023520000278949738
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,15,0.022783999641736347
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,15,0.021322667598724365
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,31,0.05286400020122528
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,31,0.03774400055408478
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,31,0.04788800080617269
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,31,0.03044266750415166
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,31,0.02934933453798294
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,31,0.029834667841593426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,31,0.02972800036271413
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,31,0.029253333806991577
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,31,0.07943999767303467
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,31,0.045968001087506614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,31,0.04186666508515676
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,31,0.028005334238211315
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,31,0.02271999915440877
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,31,0.021770666042963665
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,31,0.02161066730817159
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,31,0.021333334346612293
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,63,0.05209066470464071
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,63,0.036176001032193504
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,63,0.03159466634194056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,63,0.047279998660087585
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,63,0.02980799973011017
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,63,0.02958400050799052
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,63,0.02958400050799052
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,63,0.02994133283694585
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,63,0.07924266656239827
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,63,0.046256000796953835
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,63,0.04238933324813843
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,63,0.028101332485675812
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,63,0.023631999890009563
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,63,0.021477334201335907
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,63,0.02199999988079071
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,63,0.023178666830062866
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,127,0.054832001527150474
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,127,0.04394133388996124
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,127,0.05050666630268097
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,127,0.033626665671666466
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,127,0.029909332593282063
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,127,0.029525332152843475
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,127,0.029418667157491047
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,127,0.02940266579389572
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,127,0.047770669062932335
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,127,0.07896533111731212
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,127,0.04358399907747904
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,127,0.02757866680622101
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,127,0.023183998962243397
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,127,0.021957332889238994
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,127,0.022986667851607006
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,127,0.02160000056028366
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,255,0.06841066479682922
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,255,0.05738666653633118
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,255,0.05779199798901876
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,255,0.04623466730117798
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,255,0.04437866806983948
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,255,0.04228266576925913
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,255,0.04450666904449463
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,255,0.042090664307276406
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,255,0.08740267157554626
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,255,0.05600533386071523
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,255,0.04614399870236715
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,255,0.029989334444204967
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,255,0.023797333240509033
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,255,0.02348800003528595
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,255,0.022101332743962605
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,255,0.02176533391078313
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,511,0.09224533041318257
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,511,0.08521599570910136
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,511,0.09191466371218364
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,511,0.07357333103815715
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,511,0.06916800141334534
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,511,0.07287999987602234
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,511,0.06969599922498067
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,511,0.07069333394368489
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,511,0.09867200255393982
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,511,0.06948266426722209
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,511,0.061306665341059365
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,511,0.045968001087506614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,511,0.04062400013208389
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,511,0.040448000033696495
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,511,0.03921599934498469
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,511,0.03909866760174433
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,1023,0.14129599928855896
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,1023,0.14166399836540222
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,1023,0.1267039974530538
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,1023,0.16198933124542236
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,1023,0.12166399757067363
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,1023,0.12075199683507283
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,1023,0.12379200259844463
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,1023,0.12138666709264119
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,1023,0.1240000029404958
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,1023,0.09358400106430054
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,1023,0.09445333480834961
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,1023,0.0686773310105006
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,1023,0.06385066608587901
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,1023,0.06268266836802165
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,1023,0.06215466558933258
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,1023,0.06159999966621399
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,2047,0.243776003519694
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,2047,0.2504693269729614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,2047,0.2990506688753764
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,2047,0.23052799701690674
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,2047,0.2287999987602234
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,2047,0.22477867205937704
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,2047,0.2246346672375997
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,2047,0.22352532545725504
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,2047,0.17356799046198526
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,2047,0.15823466579119363
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,2047,0.14473066727320352
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,2047,0.11373333136240642
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,2047,0.10959466298421223
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,2047,0.10825600226720174
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,2047,0.10585600137710571
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,2047,0.10533333818117778
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,4095,0.46826668580373126
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,4095,0.42931731541951496
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,4095,0.5636533498764038
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,4095,0.44201068083445233
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,4095,0.4352426528930664
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,4095,0.4345066547393799
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,4095,0.43480531374613446
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,4095,0.4331466754277547
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,4095,0.2707573374112447
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,4095,0.24224533637364706
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,4095,0.2826346755027771
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,4095,0.2047626574834188
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,4095,0.20006400346755981
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,4095,0.19645865758260092
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,4095,0.19561066230138144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,4095,0.19686933358510336
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,8191,0.803599993387858
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,8191,0.8936533133188883
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,8191,0.8491146564483643
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,8191,1.0981067021687825
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,8191,0.8430826663970947
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,8191,0.8423893451690674
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,8191,0.839786688486735
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,8191,0.8393706480662028
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,8191,0.43507734934488934
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,8191,0.45400532086690265
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,8191,0.3850239912668864
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,8191,0.5213226477305094
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,8191,0.37985066572825116
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,8191,0.3785546620686849
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,8191,0.37586132685343426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,8191,0.37533867359161377
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,16383,1.555466651916504
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,16383,1.74726406733195
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,16383,1.6694614092508953
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,16383,2.28218666712443
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,16383,1.665013313293457
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,16383,1.6636266708374023
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,16383,1.6578559875488281
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,16383,1.6546667416890461
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,16383,0.8160479863484701
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,16383,0.8173066775004069
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,16383,1.0120373566945393
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,16383,0.7429386774698893
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,16383,0.736565351486206
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,16383,0.7341759999593099
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,16383,0.7330239613850912
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,16383,0.732485294342041
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,512,1,1,32767,3.064096132914225
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,512,1,2,32767,3.4842665990193686
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,512,1,8,32767,3.3336000442504883
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,512,1,4,32767,4.783082644144694
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,512,1,16,32767,3.333557446797689
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,512,1,32,32767,3.325450579325358
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,512,1,128,32767,3.3268372217814126
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,512,1,64,32767,3.3227465947469077
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,512,1,2,32767,1.5876000722249348
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,512,1,1,32767,1.5404319763183594
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,512,1,4,32767,2.032479921976725
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,512,1,8,32767,1.4589920043945312
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,512,1,16,32767,1.4523839950561523
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,512,1,32,32767,1.451541264851888
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,1,0.0872213343779246
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,1,0.06026133398214976
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,1,0.08516800403594971
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,512,1,64,32767,1.4492692947387695
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,1,0.052095999320348106
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,1,0.04599999884764353
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,1,0.04619200030962626
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,512,1,128,32767,1.4498987197875977
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,1,0.04606399933497111
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,1,0.04576533536116282
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,1,0.144405335187912
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,1,0.08226133386294048
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,1,0.07878399888674419
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,1,0.044112001856168113
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,1,0.03572266548871994
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,1,0.03334933271010717
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,1,0.03409066547950109
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,1,0.033743999898433685
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,3,0.08851733803749084
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,3,0.06063466767470042
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,3,0.08776533603668213
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,3,0.05266133447488149
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,3,0.04598933458328247
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,3,0.04595200220743815
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,3,0.046298667788505554
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,3,0.045893331368764244
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,3,0.1453439990679423
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,3,0.08257600168387096
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,3,0.07924266656239827
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,3,0.043578664461771645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,3,0.03640000025431315
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,3,0.034202667574087776
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,3,0.03375466664632162
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,3,0.03367999941110611
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,7,0.08736532926559448
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,7,0.061936000982920326
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,7,0.08518933256467183
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,7,0.05206400156021118
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,7,0.04613333443800608
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,7,0.04553600152333578
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,7,0.045791998505592346
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,7,0.045498669147491455
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,7,0.1439733306566874
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,7,0.08247999846935272
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,7,0.07859200239181519
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,7,0.043738668163617454
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,7,0.035904000202814736
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,7,0.033520000676314034
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,7,0.03377600014209747
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,7,0.03385599950949351
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,15,0.08827199538548787
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,15,0.06131199995676676
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,15,0.0860746701558431
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,15,0.052671998739242554
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,15,0.04637333254019419
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,15,0.04669333497683207
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,15,0.04586133360862732
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,15,0.04625066618124644
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,15,0.1448746621608734
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,15,0.08197333415349324
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,15,0.07886399825414021
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,15,0.044400001565615334
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,15,0.03561066587766012
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,15,0.033717334270477295
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,15,0.03366933266321818
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,15,0.03386666625738144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,31,0.08749333024024963
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,31,0.06196266909440359
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,31,0.08623466889063518
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,31,0.05218133330345154
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,31,0.04622933268547058
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,31,0.0469706654548645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,31,0.04619733492533366
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,31,0.046469335754712425
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,31,0.14380266269048056
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,31,0.08107733229796092
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,31,0.07939733564853668
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,31,0.044165333112080894
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,31,0.03600533306598663
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,31,0.033786666889985405
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,31,0.033957332372665405
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,31,0.03364266703526179
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,63,0.08730666836102803
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,63,0.06078400214513143
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,63,0.0879146655400594
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,63,0.052527998884518944
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,63,0.045893331368764244
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,63,0.04548266530036926
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,63,0.04613866905371348
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,63,0.045653333266576133
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,63,0.14511467019716898
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,63,0.0810346653064092
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,63,0.0792799989382426
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,63,0.03552533437808355
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,63,0.04351999859015147
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,63,0.03348266581694285
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,63,0.03422933320204417
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,63,0.0351946676770846
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,127,0.0885653297106425
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,127,0.0666133314371109
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,127,0.08874133229255676
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,127,0.058330665032068886
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,127,0.05479466418425242
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,127,0.05425066749254862
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,127,0.0543039987484614
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,127,0.05465066432952881
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,127,0.08398399750391643
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,127,0.14477333426475525
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,127,0.07900799810886383
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,127,0.04680533210436503
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,127,0.03688533355792364
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,127,0.034234667817751564
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,127,0.033759998778502144
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,127,0.03378133227427801
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,255,0.11195199688275655
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,255,0.09353066484133403
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,255,0.09874666730562846
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,255,0.07223466535409291
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,255,0.06889066596825917
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,255,0.0682826687892278
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,255,0.0687253326177597
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,255,0.06673066814740498
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,255,0.16013866662979126
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,255,0.09773866335550944
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,255,0.07922666768232982
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,255,0.04967466493447622
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,255,0.04231466849644979
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,255,0.041562666495641075
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,255,0.04012266546487808
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,255,0.04035733391841253
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,511,0.16268799702326456
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,511,0.1434506674607595
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,511,0.1673813263575236
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,511,0.12379200259844463
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,511,0.12019733587900798
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,511,0.1195146640141805
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,511,0.11618666847546895
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,511,0.11678933103879292
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,511,0.18141865730285645
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,511,0.11763733625411987
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,511,0.10812266667683919
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,511,0.07215466598669688
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,511,0.06472533444563548
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,511,0.06278400123119354
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,511,0.06087466577688853
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,511,0.06036800146102905
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,1023,0.26363199949264526
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,1023,0.24601600567499796
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,1023,0.21873599290847778
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,1023,0.30615999301274616
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,1023,0.21433067321777344
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,1023,0.21361066897710165
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,1023,0.2090346614519755
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,1023,0.20854399601618448
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,1023,0.22989332675933838
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,1023,0.16504533092180887
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,1023,0.17062399784723917
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,1023,0.11673067013422649
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,1023,0.10796266794204712
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,1023,0.10603200395901997
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,1023,0.10339200496673584
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,1023,0.10282666484514873
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,2047,0.46245332558949787
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,2047,0.45446932315826416
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,2047,0.581765333811442
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,2047,0.4190080165863037
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,2047,0.415824015935262
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,2047,0.41022932529449463
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,2047,0.4116426706314087
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,2047,0.4091413418451945
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,2047,0.3265119989713033
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,2047,0.25709333022435504
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,2047,0.2986666758855184
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,2047,0.20281066497166952
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,2047,0.1933493415514628
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,2047,0.18981333573659262
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,2047,0.1877280076344808
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,2047,0.18810667594273886
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,4095,0.8376266956329346
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,4095,0.8532266616821289
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,4095,0.808346668879191
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,4095,1.107258637746175
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,4095,0.8005599975585938
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,4095,0.8006666501363119
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,4095,0.7982666492462158
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,4095,0.7941439946492513
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,4095,0.52129065990448
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,4095,0.4355520009994507
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,4095,0.37198932965596515
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,4095,0.5450133482615153
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,4095,0.3602986733118693
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,4095,0.35679467519124347
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,4095,0.35381333033243817
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,4095,0.353216012318929
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,8191,1.5889919598897297
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,8191,1.6480053265889485
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,8191,1.5904107093811035
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,8191,2.1802560488382974
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,8191,1.5695786476135254
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,8191,1.5696533521016438
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,8191,1.5781973203023274
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,8191,1.5564853350321453
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,8191,0.8867466449737549
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,8191,0.789802630742391
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,8191,0.7041280269622803
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,8191,1.025930643081665
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,8191,0.6938079992930094
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,8191,0.6935093402862549
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,8191,0.6870453357696533
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,8191,0.6865653196970621
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,128,1024,1,1,16383,3.094773292541504
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,64,1024,1,2,16383,3.213770548502604
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,16,1024,1,8,16383,3.1400267283121743
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,32,1024,1,4,16383,4.561434745788574
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,8,1024,1,16,16383,3.1243626276652017
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,4,1024,1,32,16383,3.0889600118001304
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,2,1024,1,64,16383,3.122960090637207
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,float16,1,1024,1,128,16383,3.0878772735595703
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,128,1024,1,1,16383,1.6153066953023274
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,64,1024,1,2,16383,1.4947679837544758
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,16,1024,1,8,16383,1.3681653340657551
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,32,1024,1,4,16383,1.98689603805542
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,8,1024,1,16,16383,1.3592000007629395
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,4,1024,1,32,16383,1.353391965230306
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,2,1024,1,64,16383,1.3523786862691243
TRTLLM,1.1.0,NVIDIA GB200,mla_generation,default,float16,fp8,1,1024,1,128,16383,1.3508052825927734
