framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,1,0.012580800056457519
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,1,0.010609599947929382
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,1,0.0124208003282547
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,1,0.010739199817180634
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,1,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,1,0.010956799983978272
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,1,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,1,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,1,0.01082720011472702
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,1,0.010699199885129929
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,1,0.010766399651765823
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,1,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,1,0.010790400207042694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,1,0.010756800323724747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,3,0.012441600114107132
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,3,0.01260959953069687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,3,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,3,0.010756800323724747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,3,0.01071999967098236
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,3,0.01074720025062561
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,3,0.010873600095510482
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,3,0.01072480008006096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,3,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,3,0.01096000000834465
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,3,0.01085280030965805
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,3,0.010710400342941285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,3,0.010655999928712846
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,3,0.010694400221109391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,7,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,7,0.012620800733566284
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,7,0.010897599905729295
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,7,0.012326399981975555
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,7,0.010652799904346467
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,7,0.010809600353240967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,7,0.010795199871063232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,7,0.010787200182676315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,7,0.010831999778747558
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,7,0.010913600027561188
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,7,0.0108255997300148
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,7,0.010844799876213073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,7,0.010649599879980088
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,7,0.01088479980826378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,15,0.012851199507713318
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,15,0.011070399731397628
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,15,0.010868799686431885
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,15,0.012697599828243256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,15,0.010846400260925293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,15,0.010710400342941285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,15,0.010902400314807891
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,15,0.010793600231409073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,15,0.010873600095510482
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,15,0.010751999914646149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,15,0.011185599863529206
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,15,0.010758399963378906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,15,0.010622400045394897
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,15,0.010790400207042694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,31,0.012507200241088867
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,31,0.012414400279521943
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,31,0.010815999656915664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,31,0.010927999764680863
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,31,0.01066880002617836
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,31,0.010520000010728836
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,31,0.010833600163459777
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,31,0.010911999642848969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,31,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,31,0.010751999914646149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,31,0.010969600081443787
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,31,0.01090560033917427
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,31,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,31,0.010831999778747558
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,63,0.012763200700283051
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,63,0.010830400139093399
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,63,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,63,0.012833599746227265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,63,0.010950399935245514
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,63,0.010761599987745285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,63,0.010742399841547012
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,63,0.010956799983978272
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,63,0.010875199735164643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,63,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,63,0.010718400031328202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,63,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,63,0.010729599744081497
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,63,0.0106175996363163
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,127,0.01268479973077774
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,127,0.012724800407886505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,127,0.010856000334024429
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,127,0.010542400181293488
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,127,0.010873600095510482
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,127,0.01085439994931221
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,127,0.010831999778747558
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,127,0.010678400099277497
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,127,0.010974399745464325
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,127,0.011007999628782272
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,127,0.010702399909496308
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,127,0.010662399977445603
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,127,0.010769599676132202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,127,0.010622400045394897
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,255,0.012787200510501862
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,255,0.010784000158309937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,255,0.010873600095510482
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,255,0.011097600311040878
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,255,0.010847999900579452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,255,0.010974399745464325
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,255,0.010716799646615982
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,255,0.010735999792814255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,255,0.0108255997300148
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,255,0.010892800241708755
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,255,0.010700800269842149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,255,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,255,0.010807999968528747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,255,0.01079839989542961
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,511,0.014531199634075165
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,511,0.014412799477577209
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,511,0.012876799702644348
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,511,0.014860799908638
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,511,0.01281919926404953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,511,0.012756800651550293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,511,0.012856000661849975
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,511,0.0127920001745224
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,511,0.012716799974441528
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,511,0.012620800733566284
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,511,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,511,0.012726399302482604
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,511,0.012027200311422348
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,511,0.010667199641466141
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,1023,0.01677599996328354
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,1023,0.014742399752140044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,1023,0.014985600113868713
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,1023,0.01656160056591034
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,1023,0.015435199439525604
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,1023,0.01478240042924881
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,1023,0.01485760062932968
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,1023,0.014859199523925781
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,1023,0.014998400211334228
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,1023,0.014657600224018097
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,1023,0.012854400277137756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,1023,0.012992000579833985
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,1023,0.012972800433635712
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,1023,0.01273919939994812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,2047,0.01701440066099167
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,2047,0.016739200055599212
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,2047,0.014790399372577668
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,2047,0.016806399822235106
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,2047,0.01493919938802719
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,2047,0.014860799908638
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,2047,0.014672000706195832
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,2047,0.015478399395942689
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,2047,0.014931200444698334
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,2047,0.014764800667762756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,2047,0.012511999905109405
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,2047,0.012899200618267059
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,2047,0.014750400185585022
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,2047,0.012919999659061432
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,4095,0.01687840074300766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,4095,0.017070400714874267
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,4095,0.014827199280261993
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,4095,0.016990399360656737
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,4095,0.01483200043439865
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,4095,0.015497599542140961
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,4095,0.014856000244617463
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,4095,0.016950400173664094
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,4095,0.014921599626541137
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,4095,0.014819200336933135
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,4095,0.014745600521564484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,4095,0.014852799475193024
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,4095,0.014587199687957764
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,4095,0.014640000462532044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,8191,0.02396800071001053
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,8191,0.01983039975166321
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,8191,0.017416000366210938
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,8191,0.016859200596809388
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,8191,0.01684959977865219
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,8191,0.01688639968633652
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,8191,0.016788800060749055
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,8191,0.01937119960784912
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,8191,0.016971200704574585
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,8191,0.016977599263191222
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,8191,0.014851200580596923
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,8191,0.016918399930000307
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,8191,0.01676799952983856
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,8191,0.016782400012016297
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,16383,0.028990399837493897
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,16383,0.020960000157356263
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,16383,0.020926399528980254
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,16383,0.027166399359703063
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,16383,0.02075680047273636
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,16383,0.020980800688266753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,16383,0.020854400098323823
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,16383,0.02099999934434891
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,16383,0.023001599311828613
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,16383,0.01916320025920868
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,16383,0.01886720061302185
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,16383,0.018848000466823576
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,16383,0.018987199664115904
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,16383,0.018958400189876556
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,32767,0.03765920102596283
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,32767,0.031163200736045837
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,32767,0.029211199283599852
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,32767,0.027190399169921876
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,32767,0.027905601263046264
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,32767,0.028622400760650635
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,32767,0.028569599986076354
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,32767,0.02710399925708771
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,32767,0.02502560019493103
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,32767,0.025183999538421632
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,32767,0.022993600368499754
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,32767,0.022939200699329376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,32767,0.023054400086402894
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,32767,0.023073600232601167
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,65535,0.04018400013446808
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,65535,0.04367679953575134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,65535,0.03622080087661743
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,65535,0.035231998562812804
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,65535,0.03507040143013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,65535,0.03537279963493347
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,65535,0.03524160087108612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,65535,0.031636801362037656
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,65535,0.029185599088668822
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,65535,0.029179200530052185
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,65535,0.027105599641799927
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,65535,0.028995200991630554
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,65535,0.029051199555397034
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,65535,0.027105599641799927
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,1,131071,0.062142401933670044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,4,131071,0.05531200170516968
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,8,131071,0.05369439721107483
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,2,131071,0.07235199809074402
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,16,131071,0.054201602935791016
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,32,131071,0.057760000228881836
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1,1,64,131071,0.05571039915084839
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,1,131071,0.04036479890346527
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,2,131071,0.04187040030956268
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,4,131071,0.035155200958251955
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,8,131071,0.033220800757408145
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,16,131071,0.033529600501060484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,32,131071,0.035420799255371095
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1,1,64,131071,0.03457759916782379
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,1,0.012708799540996551
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,1,0.012699200212955475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,1,0.010815999656915664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,1,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,1,0.01077279970049858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,1,0.010784000158309937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,1,0.010847999900579452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,1,0.010875199735164643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,1,0.010788799822330475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,1,0.010683199763298035
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,1,0.010539200156927109
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,1,0.010622400045394897
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,1,0.010735999792814255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,1,0.010494399815797806
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,3,0.012745599448680877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,3,0.011289600282907486
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,3,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,3,0.010769599676132202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,3,0.010817600041627884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,3,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,3,0.01080320030450821
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,3,0.010804799944162368
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,3,0.010840000212192535
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,3,0.01075040027499199
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,3,0.010708799958229065
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,3,0.010819199681282043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,3,0.010592000186443329
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,3,0.010814400017261505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,7,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,7,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,7,0.01244639977812767
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,7,0.011004800349473954
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,7,0.01082720011472702
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,7,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,7,0.010686399787664414
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,7,0.010870400071144103
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,7,0.0107744000852108
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,7,0.010760000348091126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,7,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,7,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,7,0.010907199978828431
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,7,0.010655999928712846
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,15,0.012756800651550293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,15,0.012681600451469422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,15,0.010744000226259232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,15,0.012755200266838074
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,15,0.010784000158309937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,15,0.010907199978828431
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,15,0.010622400045394897
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,15,0.010751999914646149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,15,0.010891199856996537
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,15,0.010780800133943558
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,15,0.010689599812030793
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,15,0.010713600367307664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,15,0.010791999846696853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,15,0.01080320030450821
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,31,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,31,0.012736000120639801
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,31,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,31,0.010907199978828431
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,31,0.010896000266075134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,31,0.010729599744081497
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,31,0.01080000028014183
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,31,0.010841599851846694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,31,0.010812799632549285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,31,0.010992000252008438
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,31,0.010838399827480315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,31,0.010785599797964096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,31,0.010931199789047242
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,31,0.010782399773597717
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,63,0.01276479959487915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,63,0.012744000554084778
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,63,0.012697599828243256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,63,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,63,0.0108255997300148
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,63,0.010744000226259232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,63,0.010841599851846694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,63,0.010769599676132202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,63,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,63,0.010847999900579452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,63,0.010876800119876861
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,63,0.010635200142860412
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,63,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,127,0.01268640011548996
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,63,0.010870400071144103
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,127,0.012654399871826172
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,127,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,127,0.010916800051927567
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,127,0.01077279970049858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,127,0.010897599905729295
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,127,0.010684800148010255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,127,0.010795199871063232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,127,0.010657600313425063
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,127,0.01075040027499199
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,127,0.010779199749231338
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,127,0.01077599972486496
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,127,0.010692799836397171
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,127,0.01079839989542961
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,255,0.012678399682044983
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,255,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,255,0.012591999769210816
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,255,0.01085280030965805
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,255,0.010635200142860412
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,255,0.010780800133943558
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,255,0.010648000240325927
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,255,0.01104160025715828
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,255,0.01061599999666214
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,255,0.010876800119876861
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,255,0.010768000036478043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,255,0.01077599972486496
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,255,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,255,0.010846400260925293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,511,0.014723199605941772
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,511,0.01462399959564209
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,511,0.01379680037498474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,511,0.012899200618267059
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,511,0.01268800050020218
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,511,0.012747199833393097
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,511,0.012831999361515046
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,511,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,511,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,511,0.012745599448680877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,511,0.012646399438381195
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,511,0.01266240030527115
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,511,0.011564800143241882
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,511,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,1023,0.014847999811172486
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,1023,0.014884799718856812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,1023,0.014681600034236908
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,1023,0.01459999978542328
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,1023,0.01377120018005371
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,1023,0.012680000066757202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,1023,0.014718399941921234
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,1023,0.013012799620628356
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,1023,0.013219200074672699
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,1023,0.01276479959487915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,1023,0.012822400033473968
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,1023,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,1023,0.012807999551296235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,1023,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,2047,0.016952000558376312
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,2047,0.015017600357532501
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,2047,0.015238399803638458
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,2047,0.01467519998550415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,2047,0.014817599952220917
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,2047,0.012943999469280243
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,2047,0.014696000516414643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,2047,0.014847999811172486
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,2047,0.012859199941158295
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,2047,0.01276479959487915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,2047,0.012923200428485871
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,2047,0.012918399274349212
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,2047,0.012756800651550293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,2047,0.012671999633312225
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,4095,0.018939200043678283
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,4095,0.017745600640773775
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,4095,0.014880000054836274
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,4095,0.01464959979057312
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,4095,0.014686399698257446
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,4095,0.01276959925889969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,4095,0.014707200229167938
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,4095,0.01666239947080612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,4095,0.01679999977350235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,4095,0.012803199887275695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,4095,0.012880000472068786
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,4095,0.012907199561595917
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,4095,0.012718400359153748
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,4095,0.012777599692344665
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,8191,0.020985600352287293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,8191,0.020916800200939178
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,8191,0.018807999789714813
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,8191,0.019064000248908995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,8191,0.016873599588871004
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,8191,0.01870879977941513
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,8191,0.016847999393939973
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,8191,0.018889600038528444
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,8191,0.016812799870967864
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,8191,0.016707199811935424
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,8191,0.016859200596809388
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,8191,0.016872000694274903
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,8191,0.016385599970817566
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,8191,0.016739200055599212
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,16383,0.033323198556900024
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,16383,0.024820800125598907
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,16383,0.02218399941921234
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,16383,0.021006399393081666
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,16383,0.020931200683116914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,16383,0.020929600298404693
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,16383,0.020880000293254854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,16383,0.024380800127983094
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,16383,0.02088160067796707
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,16383,0.0189968004822731
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,16383,0.018916800618171692
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,16383,0.018848000466823576
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,16383,0.018887999653816222
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,16383,0.01887200027704239
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,32767,0.037412801384925844
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,32767,0.037083199620246886
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,32767,0.029102399945259094
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,32767,0.026974400877952574
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,32767,0.027142399549484254
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,32767,0.02606880068778992
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,32767,0.0265855997800827
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,32767,0.036904001235961915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,32767,0.026998400688171387
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,32767,0.0247296005487442
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,32767,0.024153600633144378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,32767,0.024447999894618988
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,32767,0.023145599663257597
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,32767,0.022995199263095855
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,65535,0.05901920199394226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,65535,0.05798559784889221
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,65535,0.053748798370361325
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,65535,0.05356320142745972
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,65535,0.05476639866828918
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,65535,0.059569597244262695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,65535,0.05252959728240967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,65535,0.04020319879055023
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,65535,0.03455039858818054
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,65535,0.031222400069236756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,65535,0.031142398715019226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,65535,0.029569599032402038
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,65535,0.03117760121822357
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,65535,0.03027839958667755
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,1,131071,0.08552799820899963
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,2,131071,0.09239839911460876
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,4,131071,0.07765759825706482
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,8,131071,0.07683519721031189
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,16,131071,0.07757599949836731
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,32,131071,0.07766879796981811
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,2,1,64,131071,0.07850239872932434
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,1,131071,0.056113600730895996
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,2,131071,0.0531391978263855
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,4,131071,0.04714879989624023
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,8,131071,0.04573279917240143
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,16,131071,0.04845280051231384
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,32,131071,0.04742240011692047
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,2,1,64,131071,0.0472463995218277
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,1,0.01276639997959137
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,1,0.012761600315570831
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,1,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,1,0.012451200187206269
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,1,0.011072000116109848
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,1,0.010771200060844421
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,1,0.01075040027499199
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,1,0.010828799754381179
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,1,0.010878399759531022
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,1,0.01080320030450821
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,1,0.010735999792814255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,1,0.010662399977445603
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,1,0.010871999710798264
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,1,0.010529600083827972
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,3,0.01268479973077774
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,3,0.012783999741077422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,3,0.012664000689983367
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,3,0.011416000127792359
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,3,0.010847999900579452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,3,0.010688000172376633
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,3,0.010976000130176545
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,3,0.01085119992494583
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,3,0.010718400031328202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,3,0.010806400328874588
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,3,0.010761599987745285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,3,0.010678400099277497
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,3,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,3,0.010847999900579452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,7,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,7,0.012700800597667695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,7,0.012563200294971466
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,7,0.012619200348854064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,7,0.0108255997300148
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,7,0.01077279970049858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,7,0.010699199885129929
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,7,0.011711999773979187
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,7,0.010814400017261505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,7,0.01077279970049858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,7,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,7,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,7,0.01058880016207695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,15,0.012745599448680877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,7,0.010751999914646149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,15,0.012625600397586822
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,15,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,15,0.011716800183057785
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,15,0.010896000266075134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,15,0.010751999914646149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,15,0.010625600069761276
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,15,0.010745599865913391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,15,0.010793600231409073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,15,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,15,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,15,0.0106175996363163
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,15,0.010790400207042694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,15,0.010918399691581726
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,31,0.01273760050535202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,31,0.011979199945926666
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,31,0.012734399735927581
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,31,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,31,0.010735999792814255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,31,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,31,0.01085280030965805
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,31,0.012411200255155564
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,31,0.010679999738931656
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,31,0.010740800201892853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,31,0.010993599891662598
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,31,0.010768000036478043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,31,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,31,0.010527999699115753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,63,0.012680000066757202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,63,0.012604799866676331
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,63,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,63,0.01080000028014183
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,63,0.011427199840545655
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,63,0.01080000028014183
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,63,0.010675200074911118
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,63,0.010913600027561188
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,63,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,63,0.01074720025062561
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,63,0.01064160019159317
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,63,0.010662399977445603
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,63,0.010639999806880952
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,63,0.010758399963378906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,127,0.012708799540996551
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,127,0.012774400413036346
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,127,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,127,0.0125231996178627
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,127,0.0107744000852108
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,127,0.010876800119876861
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,127,0.010679999738931656
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,127,0.010806400328874588
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,127,0.010820800065994262
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,127,0.010566399991512298
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,127,0.01117440015077591
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,127,0.010745599865913391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,127,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,127,0.0108255997300148
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,255,0.012987199425697326
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,255,0.012579199671745301
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,255,0.011523199826478958
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,255,0.012652799487113953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,255,0.010633599758148194
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,255,0.010921599715948105
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,255,0.010768000036478043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,255,0.011865600198507308
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,255,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,255,0.011657600104808808
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,255,0.010742399841547012
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,255,0.010649599879980088
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,255,0.010574399679899215
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,255,0.010849600285291671
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,511,0.01467680037021637
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,511,0.014774399995803832
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,511,0.014644800126552582
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,511,0.014795200526714325
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,511,0.012785600125789642
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,511,0.01266079992055893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,511,0.012870399653911591
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,511,0.012833599746227265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,511,0.012771199643611907
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,511,0.012678399682044983
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,511,0.012801599502563477
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,511,0.012753599882125854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,511,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,511,0.01268479973077774
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,1023,0.01686079949140549
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,1023,0.014841599762439728
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,1023,0.014774399995803832
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,1023,0.014694400131702423
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,1023,0.01465280055999756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,1023,0.012779200077056884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,1023,0.01390880048274994
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,1023,0.015430399775505066
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,1023,0.014497600495815277
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,1023,0.012742400169372559
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,1023,0.012705600261688233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,1023,0.012907199561595917
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,1023,0.012875199317932129
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,1023,0.012692800164222718
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,2047,0.017182399332523347
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,2047,0.01690559983253479
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,2047,0.0150736004114151
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,2047,0.014763200283050537
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,2047,0.014822399616241455
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,2047,0.01496800035238266
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,2047,0.014295999705791474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,2047,0.014787200093269347
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,2047,0.014747199416160584
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,2047,0.012923200428485871
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,2047,0.012742400169372559
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,2047,0.012591999769210816
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,2047,0.012644800543785095
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,2047,0.012865599989891053
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,4095,0.020977599918842314
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,4095,0.018777599930763243
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,4095,0.01886560022830963
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,4095,0.01703840047121048
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,4095,0.01671839952468872
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,4095,0.01677920073270798
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,4095,0.0186271995306015
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,4095,0.01671999990940094
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,4095,0.016884799301624297
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,4095,0.016782400012016297
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,4095,0.014844800531864166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,4095,0.014841599762439728
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,4095,0.014948800206184387
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,4095,0.014822399616241455
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,8191,0.03107840120792389
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,8191,0.020948800444602966
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,8191,0.02091519981622696
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,8191,0.01882079988718033
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,8191,0.01880960017442703
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,8191,0.018904000520706177
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,8191,0.01886560022830963
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,8191,0.02316479980945587
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,8191,0.018731200695037843
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,8191,0.016896000504493712
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,8191,0.016944000124931337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,8191,0.016740800440311433
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,8191,0.016835199296474458
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,8191,0.01679839938879013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,16383,0.03201119899749756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,16383,0.03546879887580871
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,16383,0.02359520047903061
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,16383,0.023812800645828247
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,16383,0.023105600476264955
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,16383,0.02284799963235855
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,16383,0.022944000363349915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,16383,0.029526400566101074
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,16383,0.024747200310230255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,16383,0.020883199572563172
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,16383,0.020838400721549986
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,16383,0.021104000508785248
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,16383,0.01906079947948456
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,16383,0.01879040002822876
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,32767,0.05235999822616577
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,32767,0.05395680069923401
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,32767,0.04970720112323761
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,32767,0.04831520020961762
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,32767,0.049563199281692505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,32767,0.04955680072307587
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,32767,0.04829919934272766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,32767,0.03372479975223541
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,32767,0.03206560015678406
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,32767,0.026982399821281432
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,32767,0.0271232008934021
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,32767,0.02507199943065643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,32767,0.02492479979991913
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,32767,0.025488001108169556
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,65535,0.08050079941749573
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,65535,0.09065759778022767
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,65535,0.07279199957847596
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,65535,0.07207679748535156
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,65535,0.07244480252265931
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,65535,0.07172639966011048
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,65535,0.07185119986534119
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,65535,0.04951040148735046
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,65535,0.05178719758987427
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,65535,0.0430400013923645
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,65535,0.04139359891414642
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,65535,0.04302079975605011
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,65535,0.04193759858608246
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,65535,0.04150559902191162
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,1,131071,0.13276959657669068
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,2,131071,0.1600607991218567
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,4,131071,0.12384480237960815
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,8,131071,0.12178560495376586
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,16,131071,0.12315360307693482
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,32,131071,0.12152960300445556
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,4,1,64,131071,0.12251520156860352
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,1,131071,0.07480800151824951
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,2,131071,0.08488960266113281
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,4,131071,0.06404799818992615
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,8,131071,0.06418079733848572
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,16,131071,0.06411359906196594
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,32,131071,0.06400160193443298
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,4,1,64,131071,0.06374559998512268
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,1,0.012894399464130402
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,1,0.012630400061607362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,1,0.012697599828243256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,1,0.01266079992055893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,1,0.012727999687194824
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,1,0.0108815997838974
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,1,0.010758399963378906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,1,0.012604799866676331
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,1,0.011417599767446518
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,1,0.01072319969534874
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,1,0.01082720011472702
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,1,0.01069760024547577
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,1,0.010755199939012527
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,1,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,3,0.01295199990272522
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,3,0.01260959953069687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,3,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,3,0.012876799702644348
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,3,0.01072160005569458
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,3,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,3,0.01148959994316101
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,3,0.012588800489902496
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,3,0.010892800241708755
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,3,0.010811199992895126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,3,0.010611200332641601
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,3,0.010768000036478043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,3,0.010678400099277497
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,3,0.010820800065994262
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,7,0.012787200510501862
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,7,0.012694400548934937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,7,0.012691199779510498
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,7,0.012726399302482604
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,7,0.01072480008006096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,7,0.011374399811029435
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,7,0.010830400139093399
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,7,0.0126351997256279
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,7,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,7,0.01088479980826378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,7,0.010678400099277497
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,7,0.010868799686431885
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,7,0.010547199845314026
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,7,0.010838399827480315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,15,0.012726399302482604
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,15,0.012630400061607362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,15,0.012681600451469422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,15,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,15,0.01244800016283989
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,15,0.01074720025062561
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,15,0.012908799946308136
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,15,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,15,0.010704000294208527
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,15,0.010864000022411346
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,15,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,15,0.010735999792814255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,15,0.010758399963378906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,15,0.010785599797964096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,31,0.012783999741077422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,31,0.012703999876976013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,31,0.0126351997256279
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,31,0.012675200402736665
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,31,0.01260959953069687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,31,0.010692799836397171
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,31,0.011001600325107575
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,31,0.012699200212955475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,31,0.012243200093507767
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,31,0.010796800255775452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,31,0.010700800269842149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,31,0.010847999900579452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,31,0.010745599865913391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,31,0.01061440035700798
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,63,0.01273919939994812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,63,0.012731200456619263
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,63,0.012774400413036346
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,63,0.010777600109577179
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,63,0.011206399649381638
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,63,0.010870400071144103
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,63,0.010764800012111664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,63,0.012750400602817536
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,63,0.010745599865913391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,63,0.010787200182676315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,63,0.01067200005054474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,63,0.010769599676132202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,63,0.010676799714565277
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,63,0.010844799876213073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,127,0.012875199317932129
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,127,0.01273919939994812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,127,0.011958400160074234
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,127,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,127,0.0126351997256279
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,127,0.010648000240325927
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,127,0.010940799862146378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,127,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,127,0.010676799714565277
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,127,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,127,0.010819199681282043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,127,0.010844799876213073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,127,0.010694400221109391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,127,0.010766399651765823
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,255,0.012838399410247803
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,255,0.012727999687194824
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,255,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,255,0.011929599940776825
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,255,0.012532800436019897
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,255,0.010815999656915664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,255,0.010644800215959548
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,255,0.012664000689983367
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,255,0.010942400246858597
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,255,0.010865599662065507
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,255,0.010793600231409073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,255,0.010635200142860412
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,255,0.010744000226259232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,255,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,511,0.014787200093269347
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,511,0.014761599898338317
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,511,0.01472959965467453
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,511,0.014684799313545226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,511,0.0147024005651474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,511,0.012716799974441528
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,511,0.012886400520801543
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,511,0.012884800136089326
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,511,0.01266240030527115
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,511,0.012700800597667695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,511,0.012675200402736665
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,511,0.0125231996178627
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,511,0.012574400007724761
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,511,0.012692800164222718
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,1023,0.017761600017547608
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,1023,0.01688639968633652
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,1023,0.014740799367427827
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,1023,0.014852799475193024
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,1023,0.014873600006103516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,1023,0.014632000029087067
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,1023,0.014614400267601014
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,1023,0.014723199605941772
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,1023,0.014723199605941772
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,1023,0.014635199308395385
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,1023,0.012732799351215362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,1023,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,1023,0.012838399410247803
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,2047,0.01878879964351654
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,1023,0.012774400413036346
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,2047,0.020873600244522096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,2047,0.017046399414539337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,2047,0.016840000450611115
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,2047,0.016948799788951873
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,2047,0.01674239933490753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,2047,0.01674560010433197
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,2047,0.016780799627304076
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,2047,0.014849600195884705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,2047,0.014737600088119506
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,2047,0.014750400185585022
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,2047,0.013044799864292144
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,2047,0.013633599877357483
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,2047,0.012652799487113953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,4095,0.029180800914764403
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,4095,0.020868800580501556
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,4095,0.019070400297641753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,4095,0.019019199907779692
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,4095,0.01884640008211136
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,4095,0.016952000558376312
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,4095,0.017136000096797943
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,4095,0.021129600703716278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,4095,0.017763200402259826
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,4095,0.016752000153064727
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,4095,0.016840000450611115
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,4095,0.014787200093269347
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,4095,0.014867199957370758
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,4095,0.014878399670124054
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,8191,0.030089598894119263
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,8191,0.03146399855613709
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,8191,0.023024000227451324
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,8191,0.021004800498485566
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,8191,0.021784000098705292
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,8191,0.020937600731849672
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,8191,0.020982399582862854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,8191,0.025070399045944214
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,8191,0.022937600314617158
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,8191,0.018750399351119995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,8191,0.0188511997461319
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,8191,0.017430399358272553
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,8191,0.016967999935150146
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,8191,0.016811199486255646
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,16383,0.04989440143108368
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,16383,0.05634719729423523
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,16383,0.04756639897823334
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,16383,0.046054399013519286
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,16383,0.046254399418830874
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,16383,0.0456959992647171
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,16383,0.04587680101394653
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,16383,0.029398399591445922
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,16383,0.032867199182510375
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,16383,0.02503040134906769
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,16383,0.023689599335193635
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,16383,0.02492000013589859
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,16383,0.02295839935541153
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,16383,0.023044799268245698
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,32767,0.07537760138511658
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,32767,0.09321920275688171
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,32767,0.06964319944381714
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,32767,0.06968320012092591
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,32767,0.06881759762763977
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,32767,0.07009440064430236
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,32767,0.06863679885864257
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,32767,0.04749439954757691
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,32767,0.051718401908874514
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,32767,0.040110400319099425
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,32767,0.03933280110359192
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,32767,0.03993119895458221
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,32767,0.03933280110359192
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,32767,0.03849599957466125
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,65535,0.1292880058288574
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,65535,0.16421120166778563
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,65535,0.11920479536056519
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,65535,0.12051039934158325
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,65535,0.11943520307540893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,65535,0.12083200216293336
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,65535,0.11709920167922974
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,65535,0.07177280187606812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,65535,0.08457760214805603
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,65535,0.06253119707107543
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,65535,0.060014402866363524
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,65535,0.06123359799385071
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,65535,0.05992479920387268
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,65535,0.06013280153274536
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,1,131071,0.2327791929244995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,4,131071,0.21708159446716307
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,2,131071,0.3021183967590332
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,8,131071,0.21788320541381836
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,16,131071,0.21766080856323242
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,32,131071,0.2154320001602173
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,8,1,64,131071,0.21745600700378417
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,1,131071,0.11970560550689698
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,2,131071,0.14812480211257933
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,4,131071,0.10480959415435791
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,8,131071,0.10362720489501953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,16,131071,0.10494719743728638
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,32,131071,0.10475200414657593
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,8,1,64,131071,0.10291039943695068
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,1,0.01284639984369278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,1,0.012782399356365205
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,1,0.012718400359153748
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,1,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,1,0.012689599394798278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,1,0.012668800354003907
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,1,0.010689599812030793
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,1,0.012779200077056884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,1,0.012671999633312225
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,1,0.010841599851846694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,1,0.010585600137710571
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,1,0.010632000118494033
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,1,0.01088000014424324
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,1,0.010795199871063232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,3,0.012779200077056884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,3,0.012591999769210816
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,3,0.012806400656700134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,3,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,3,0.012508800625801087
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,3,0.010548800230026245
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,3,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,3,0.012839999794960023
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,3,0.012851199507713318
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,3,0.010780800133943558
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,3,0.010907199978828431
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,3,0.010780800133943558
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,3,0.010940799862146378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,3,0.010704000294208527
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,7,0.012889599800109864
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,7,0.012763200700283051
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,7,0.012761600315570831
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,7,0.0126351997256279
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,7,0.012715199589729309
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,7,0.012624000012874604
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,7,0.011879999935626984
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,7,0.012641599774360657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,7,0.012692800164222718
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,7,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,7,0.010923200100660325
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,7,0.0106175996363163
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,7,0.010828799754381179
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,7,0.01072480008006096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,15,0.01289760023355484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,15,0.012732799351215362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,15,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,15,0.012628799676895142
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,15,0.012641599774360657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,15,0.011228799819946289
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,15,0.010843200236558914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,15,0.012736000120639801
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,15,0.01290079951286316
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,15,0.01154239997267723
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,15,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,15,0.011023999750614166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,15,0.010740800201892853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,15,0.010836800187826156
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,31,0.012748800218105316
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,31,0.012936000525951386
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,31,0.012703999876976013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,31,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,31,0.012415999919176102
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,31,0.012744000554084778
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,31,0.010817600041627884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,31,0.01266240030527115
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,31,0.012665599584579468
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,31,0.010795199871063232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,31,0.010689599812030793
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,31,0.010744000226259232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,31,0.010916800051927567
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,31,0.01082720011472702
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,63,0.012881599366664886
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,63,0.012736000120639801
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,63,0.012624000012874604
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,63,0.012721599638462066
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,63,0.01080159991979599
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,63,0.01271360069513321
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,63,0.01271039992570877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,63,0.012831999361515046
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,63,0.011204800009727478
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,63,0.01085119992494583
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,63,0.010726399719715118
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,63,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,63,0.010755199939012527
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,63,0.010840000212192535
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,127,0.01462559998035431
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,127,0.01281919926404953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,127,0.012665599584579468
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,127,0.012801599502563477
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,127,0.012619200348854064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,127,0.012366399914026261
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,127,0.010785599797964096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,127,0.012671999633312225
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,127,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,127,0.010700800269842149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,127,0.010715200006961823
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,127,0.010769599676132202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,127,0.010804799944162368
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,127,0.010667199641466141
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,255,0.012595200538635254
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,255,0.012649600207805634
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,255,0.012736000120639801
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,255,0.012652799487113953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,255,0.010787200182676315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,255,0.01263359934091568
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,255,0.010867200046777725
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,255,0.012673600018024445
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,255,0.011695999652147293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,255,0.010742399841547012
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,255,0.010793600231409073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,255,0.010675200074911118
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,255,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,255,0.010840000212192535
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,511,0.016889600455760954
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,511,0.014839999377727509
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,511,0.0147024005651474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,511,0.014875200390815736
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,511,0.014612799882888794
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,511,0.013918399810791016
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,511,0.012862400710582733
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,511,0.014691199362277984
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,511,0.012873600423336028
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,511,0.012699200212955475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,511,0.012683199346065521
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,511,0.012723200023174286
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,511,0.012753599882125854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,511,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,1023,0.020849600434303284
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,1023,0.016816000640392303
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,1023,0.016814400255680085
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,1023,0.015662400424480437
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,1023,0.015806399285793304
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,1023,0.014892800152301789
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,1023,0.016064000129699708
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,1023,0.016715200245380403
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,1023,0.01478559970855713
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,1023,0.014825600385665893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,1023,0.014006400108337402
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,1023,0.012753599882125854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,1023,0.012777599692344665
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,1023,0.01281599998474121
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,2047,0.02715519964694977
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,2047,0.020919999480247496
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,2047,0.018731200695037843
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,2047,0.018799999356269838
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,2047,0.018857599794864656
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,2047,0.018694399297237395
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,2047,0.01690399944782257
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,2047,0.020972800254821778
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,2047,0.016843199729919434
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,2047,0.014748799800872802
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,2047,0.014828799664974213
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,2047,0.014891199767589569
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,2047,0.014697599411010741
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,2047,0.014696000516414643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,4095,0.029151999950408937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,4095,0.030019199848175047
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,4095,0.02098879963159561
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,4095,0.02091519981622696
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,4095,0.020950399339199066
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,4095,0.020585599541664123
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,4095,0.018952000141143798
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,4095,0.02301599979400635
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,4095,0.020878399908542632
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,4095,0.01690080016851425
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,4095,0.016892799735069276
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,4095,0.016888000071048737
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,4095,0.016659200191497803
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,4095,0.016729600727558136
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,8191,0.05074080228805542
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,8191,0.053732800483703616
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,8191,0.04551840126514435
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,8191,0.0455487996339798
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,8191,0.044177600741386415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,8191,0.04380640089511871
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,8191,0.04422720074653626
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,8191,0.029099199175834655
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,8191,0.029039999842643736
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,8191,0.022921599447727203
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,8191,0.02295680046081543
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,8191,0.021057599782943727
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,8191,0.02098879963159561
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,8191,0.02099519968032837
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,16383,0.07686399817466735
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,16383,0.09332000017166138
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,16383,0.07128480076789856
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,16383,0.07067360281944275
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,16383,0.07239199876785278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,16383,0.069513601064682
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,16383,0.06907839775085449
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,16383,0.04624800086021423
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,16383,0.05086399912834168
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,16383,0.03965280055999756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,16383,0.03939839899539947
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,16383,0.039366400241851805
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,16383,0.03924640119075775
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,16383,0.03883039951324463
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,32767,0.12778079509735107
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,32767,0.17089279890060424
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,32767,0.12421760559082032
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,32767,0.12196160554885864
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,32767,0.12029600143432617
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,32767,0.11987680196762085
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,32767,0.12125760316848755
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,32767,0.07062079906463622
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,32767,0.08615360260009766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,32767,0.06226239800453186
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,32767,0.06191200017929077
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,32767,0.061084800958633424
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,32767,0.06082879900932312
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,32767,0.06028640270233154
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,65535,0.23069119453430176
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,65535,0.3231791973114014
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,65535,0.21726720333099364
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,65535,0.21728160381317138
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,65535,0.21744799613952637
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,65535,0.21562559604644777
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,65535,0.2166383981704712
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,65535,0.11509920358657837
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,65535,0.10550559759140014
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,65535,0.1584239959716797
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,65535,0.103547203540802
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,65535,0.10320320129394531
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,65535,0.10333440303802491
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,65535,0.10324640274047851
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,1,131071,0.42570080757141116
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,2,131071,0.6245776176452636
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,4,131071,0.40617918968200684
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,8,131071,0.4089503765106201
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,16,131071,0.4052015781402588
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,32,131071,0.4045584201812744
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,16,1,64,131071,0.4046175956726074
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,1,131071,0.2108016014099121
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,2,131071,0.2974623918533325
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,4,131071,0.1866320013999939
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,8,131071,0.18528159856796264
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,16,131071,0.18587520122528076
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,32,131071,0.18417439460754395
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,1,0.014659200608730317
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,16,1,64,131071,0.18336160182952882
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,1,0.012860800325870513
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,1,0.012883199751377106
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,1,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,1,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,1,0.012833599746227265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,1,0.012807999551296235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,1,0.014609600603580474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,1,0.012695999443531036
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,1,0.012803199887275695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,1,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,1,0.012556800246238708
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,1,0.012544000148773193
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,1,0.012692800164222718
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,3,0.0146479994058609
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,3,0.012860800325870513
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,3,0.012785600125789642
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,3,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,3,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,3,0.012902399897575379
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,3,0.012702399492263794
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,3,0.01273919939994812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,3,0.012783999741077422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,3,0.013014400005340576
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,3,0.012619200348854064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,3,0.01077599972486496
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,3,0.012620800733566284
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,3,0.012540799379348756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,7,0.01486240029335022
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,7,0.01284639984369278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,7,0.012824000418186187
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,7,0.012731200456619263
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,7,0.012796799838542938
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,7,0.012702399492263794
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,7,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,7,0.012873600423336028
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,7,0.01276639997959137
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,7,0.01175839975476265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,7,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,7,0.010694400221109391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,7,0.010945600271224976
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,7,0.010713600367307664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,15,0.014689600467681885
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,15,0.014403200149536133
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,15,0.012881599366664886
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,15,0.012716799974441528
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,15,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,15,0.012783999741077422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,15,0.012697599828243256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,15,0.012742400169372559
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,15,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,15,0.012580800056457519
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,15,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,15,0.010784000158309937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,15,0.010716799646615982
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,31,0.014894400537014008
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,31,0.013254399597644805
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,15,0.011670400202274323
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,31,0.012796799838542938
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,31,0.012873600423336028
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,31,0.012750400602817536
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,31,0.012849600613117218
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,31,0.012756800651550293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,31,0.014734399318695069
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,31,0.012833599746227265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,31,0.012705600261688233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,31,0.010788799822330475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,31,0.012630400061607362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,31,0.010817600041627884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,31,0.010948800295591355
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,63,0.015000000596046448
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,63,0.014867199957370758
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,63,0.012881599366664886
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,63,0.01295199990272522
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,63,0.012912000715732574
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,63,0.012988799810409546
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,63,0.012740799784660339
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,63,0.012886400520801543
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,63,0.01271360069513321
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,63,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,63,0.010769599676132202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,63,0.011116799712181092
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,63,0.010811199992895126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,63,0.011457599699497223
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,127,0.014763200283050537
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,127,0.014748799800872802
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,127,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,127,0.01276479959487915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,127,0.012654399871826172
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,127,0.012887999415397644
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,127,0.012875199317932129
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,127,0.012848000228404998
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,127,0.012692800164222718
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,127,0.012598399817943574
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,127,0.012639999389648438
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,127,0.01074720025062561
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,127,0.010700800269842149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,127,0.0107744000852108
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,255,0.014894400537014008
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,255,0.012750400602817536
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,255,0.01273919939994812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,255,0.012681600451469422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,255,0.012775999307632447
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,255,0.012619200348854064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,255,0.012718400359153748
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,255,0.01276959925889969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,255,0.012617599964141846
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,255,0.012628799676895142
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,255,0.010704000294208527
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,255,0.011931200325489045
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,255,0.01079839989542961
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,255,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,511,0.018937599658966065
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,511,0.016766400635242464
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,511,0.01478240042924881
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,511,0.014737600088119506
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,511,0.015015999972820281
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,511,0.014875200390815736
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,511,0.014847999811172486
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,511,0.014841599762439728
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,511,0.01465280055999756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,511,0.012839999794960023
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,511,0.012848000228404998
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,511,0.012747199833393097
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,511,0.012899200618267059
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,511,0.012702399492263794
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,1023,0.025003200769424437
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,1023,0.020899200439453126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,1023,0.016971200704574585
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,1023,0.01690080016851425
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,1023,0.016782400012016297
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,1023,0.01677280068397522
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,1023,0.01687040030956268
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,1023,0.01894560009241104
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,1023,0.01671680063009262
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,1023,0.014849600195884705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,1023,0.014752000570297241
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,1023,0.014662399888038635
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,1023,0.014707200229167938
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,1023,0.014526399970054626
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,2047,0.02720159888267517
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,2047,0.031385600566864014
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,2047,0.02115200012922287
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,2047,0.019993600249290467
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,2047,0.020931200683116914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,2047,0.02093279957771301
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,2047,0.01947840005159378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,2047,0.02083359956741333
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,2047,0.020977599918842314
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,2047,0.01681919991970062
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,2047,0.014846399426460266
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,2047,0.014824000000953675
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,2047,0.01488800048828125
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,2047,0.015009599924087524
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,4095,0.04976640045642853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,4095,0.055542397499084475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,4095,0.04327360093593598
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,4095,0.041947200894355774
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,4095,0.04199999868869782
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,4095,0.04286080002784729
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,4095,0.0428384006023407
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,4095,0.026651200652122498
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,4095,0.02916640043258667
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,4095,0.020927999913692475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,4095,0.018911999464035035
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,4095,0.020744000375270844
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,4095,0.01889760047197342
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,4095,0.018958400189876556
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,8191,0.07489600181579589
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,8191,0.09467840194702148
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,8191,0.06924639940261841
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,8191,0.06596320271492004
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,8191,0.0691648006439209
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,8191,0.06839039921760559
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,8191,0.0683184027671814
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,8191,0.045705598592758176
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,8191,0.051894402503967284
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,8191,0.037643200159072875
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,8191,0.03715519905090332
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,8191,0.035835200548172
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,8191,0.03654400110244751
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,8191,0.03648160099983215
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,16383,0.17133760452270508
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,16383,0.12305760383605957
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,16383,0.1348639965057373
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,16383,0.12281759977340698
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,16383,0.12001919746398926
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,16383,0.12153439521789551
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,16383,0.12297279834747314
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,16383,0.07117919921875
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,16383,0.08734719753265381
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,16383,0.06004959940910339
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,16383,0.05984640121459961
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,16383,0.059329599142074585
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,16383,0.05856159925460815
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,16383,0.058087998628616334
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,32767,0.23906240463256836
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,32767,0.32486560344696047
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,32767,0.22308640480041503
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,32767,0.2219007968902588
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,32767,0.22168800830841065
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,32767,0.2222912073135376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,32767,0.22068159580230712
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,32767,0.12065279483795166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,32767,0.16008319854736328
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,32767,0.10421600341796874
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,32767,0.10300960540771484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,32767,0.1027184009552002
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,32767,0.10327680110931396
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,32767,0.10298399925231934
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,1,65535,0.4418511867523193
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,2,65535,0.6301887989044189
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,4,65535,0.42499518394470215
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,8,65535,0.42214398384094237
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,16,65535,0.42297921180725095
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,32,65535,0.42287840843200686
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,32,1,64,65535,0.42252001762390134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,1,65535,0.21924960613250732
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,2,65535,0.29549920558929443
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,4,65535,0.19365600347518921
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,8,65535,0.19136159420013427
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,16,65535,0.19115359783172609
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,32,65535,0.19002079963684082
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,32,1,64,65535,0.19197920560836793
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,1,0.014734399318695069
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,1,0.014688000082969666
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,1,0.012800000607967377
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,1,0.012785600125789642
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,1,0.012852799892425538
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,1,0.012694400548934937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,1,0.01271039992570877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,1,0.013355199992656708
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,1,0.0146479994058609
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,1,0.012668800354003907
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,1,0.012755200266838074
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,1,0.012734399735927581
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,1,0.01271360069513321
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,1,0.012758399546146392
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,3,0.014716799557209014
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,3,0.014839999377727509
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,3,0.012824000418186187
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,3,0.012759999930858612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,3,0.012745599448680877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,3,0.012902399897575379
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,3,0.012744000554084778
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,3,0.012908799946308136
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,3,0.012828800082206725
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,3,0.012759999930858612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,3,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,3,0.011110399663448334
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,3,0.011427199840545655
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,3,0.01265919953584671
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,7,0.014705599844455719
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,7,0.014766399562358857
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,7,0.013313600420951843
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,7,0.01273760050535202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,7,0.012664000689983367
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,7,0.012692800164222718
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,7,0.012862400710582733
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,7,0.012800000607967377
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,7,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,7,0.012700800597667695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,7,0.012580800056457519
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,7,0.012798400223255157
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,7,0.010569600015878677
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,7,0.011222399771213531
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,15,0.014710399508476257
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,15,0.014827199280261993
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,15,0.01271200031042099
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,15,0.012779200077056884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,15,0.01273760050535202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,15,0.012775999307632447
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,15,0.01276959925889969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,15,0.012849600613117218
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,15,0.013510400056838989
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,15,0.01257600039243698
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,15,0.012587200105190276
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,15,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,15,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,31,0.01448799967765808
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,15,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,31,0.014844800531864166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,31,0.012822400033473968
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,31,0.01276639997959137
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,31,0.013575999438762665
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,31,0.012747199833393097
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,31,0.01276959925889969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,31,0.012876799702644348
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,31,0.014776000380516052
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,31,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,31,0.011444800347089768
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,31,0.012651200592517852
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,31,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,31,0.010688000172376633
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,63,0.01467519998550415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,63,0.014955200254917145
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,63,0.014660799503326416
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,63,0.01279360055923462
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,63,0.012708799540996551
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,63,0.013059200346469879
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,63,0.012870399653911591
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,63,0.012768000364303589
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,63,0.012931199371814727
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,63,0.012836800515651703
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,63,0.012617599964141846
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,63,0.01268800050020218
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,63,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,63,0.012486399710178375
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,127,0.014742399752140044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,127,0.014726400375366211
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,127,0.013982400298118591
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,127,0.013921600580215455
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,127,0.012807999551296235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,127,0.01276959925889969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,127,0.012998400628566742
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,127,0.012759999930858612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,127,0.012787200510501862
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,127,0.012612800300121307
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,127,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,127,0.012630400061607362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,127,0.01265919953584671
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,127,0.010830400139093399
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,255,0.015187199413776397
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,255,0.014900800585746766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,255,0.013120000064373017
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,255,0.01271039992570877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,255,0.012825599312782288
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,255,0.012916800379753113
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,255,0.01273919939994812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,255,0.01475519984960556
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,255,0.012875199317932129
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,255,0.012771199643611907
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,255,0.011078400164842605
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,255,0.010873600095510482
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,255,0.012383999675512314
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,255,0.01266079992055893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,511,0.023004800081253052
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,511,0.01882079988718033
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,511,0.016787199676036833
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,511,0.01674399971961975
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,511,0.014985600113868713
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,511,0.016462400555610657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,511,0.016828800737857818
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,511,0.020870399475097657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,511,0.014734399318695069
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,511,0.013422399759292603
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,511,0.012825599312782288
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,511,0.012868799269199371
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,511,0.012744000554084778
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,1023,0.02922559976577759
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,511,0.012604799866676331
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,1023,0.02954080104827881
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,1023,0.019303999841213226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,1023,0.019257600605487823
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,1023,0.018935999274253844
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,1023,0.019075199961662292
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,1023,0.018772800266742707
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,1023,0.022910399734973906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,1023,0.018932799994945525
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,1023,0.017052799463272095
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,1023,0.014819200336933135
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,1023,0.014876799285411834
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,1023,0.014705599844455719
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,1023,0.014884799718856812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,2047,0.05173919796943664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,2047,0.05378559827804565
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,2047,0.043742400407791135
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,2047,0.04348160028457641
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,2047,0.04328480064868927
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,2047,0.043412798643112184
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,2047,0.04210880100727081
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,2047,0.02909280061721802
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,2047,0.028543999791145323
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,2047,0.02096800059080124
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,2047,0.019543999433517457
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,2047,0.01903519928455353
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,2047,0.018884800374507904
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,2047,0.018916800618171692
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,4095,0.0782480001449585
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,4095,0.09224799871444703
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,4095,0.07018719911575318
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,4095,0.06875839829444885
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,4095,0.06802560091018676
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,4095,0.0685375988483429
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,4095,0.06846240162849426
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,4095,0.04898079931735992
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,4095,0.05013120174407959
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,4095,0.039647999405860904
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,4095,0.03730080127716064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,4095,0.037415999174118045
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,4095,0.03591679930686951
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,4095,0.037355199456214905
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,8191,0.13577920198440552
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,8191,0.16920000314712524
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,8191,0.12439039945602418
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,8191,0.122542405128479
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,8191,0.12162079811096191
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,8191,0.12282240390777588
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,8191,0.12169760465621948
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,8191,0.07464320063591004
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,8191,0.08524640202522278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,8191,0.06206240057945252
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,8191,0.05893440246582031
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,8191,0.05956159830093384
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,8191,0.058169597387313844
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,8191,0.05940960049629211
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,16383,0.23797919750213622
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,16383,0.22305760383605958
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,16383,0.3188447952270508
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,16383,0.22293760776519775
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,16383,0.22186079025268554
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,16383,0.22052640914916993
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,16383,0.22368319034576417
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,16383,0.12310080528259278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,16383,0.15473920106887817
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,16383,0.1053920030593872
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,16383,0.10340800285339355
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,16383,0.10313279628753662
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,16383,0.10240479707717895
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,16383,0.10272959470748902
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,1,32767,0.44852638244628906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,2,32767,0.6183263778686523
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,4,32767,0.4251664161682129
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,8,32767,0.4202256202697754
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,16,32767,0.41909117698669435
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,32,32767,0.42143521308898924
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,64,1,64,32767,0.42192959785461426
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,1,32767,0.2222752094268799
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,2,32767,0.29930720329284666
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,4,32767,0.19364479780197144
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,8,32767,0.1908128023147583
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,16,32767,0.1913648009300232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,32,32767,0.19124799966812134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,64,1,64,32767,0.19054880142211914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,1,0.01693280041217804
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,1,0.020923200249671935
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,1,0.014758400619029999
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,1,0.014721600711345673
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,1,0.014361600577831268
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,1,0.014739200472831726
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,1,0.014595200121402741
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,1,0.01669120043516159
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,1,0.018212799727916718
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,1,0.0147024005651474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,1,0.012590399384498597
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,1,0.012641599774360657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,1,0.012580800056457519
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,1,0.012649600207805634
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,3,0.016918399930000307
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,3,0.02090719938278198
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,3,0.014865599572658539
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,3,0.014723199605941772
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,3,0.014632000029087067
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,3,0.014694400131702423
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,3,0.013761599361896516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,3,0.018771199882030486
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,3,0.012761600315570831
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,3,0.016920000314712524
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,3,0.012603199481964112
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,3,0.01260959953069687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,3,0.01265919953584671
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,3,0.012697599828243256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,7,0.016977599263191222
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,7,0.02099840044975281
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,7,0.01520799994468689
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,7,0.01478080004453659
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,7,0.014683200418949128
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,7,0.014683200418949128
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,7,0.014681600034236908
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,7,0.01684480011463165
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,7,0.018225599825382233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,7,0.012857599556446076
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,7,0.012699200212955475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,7,0.01252799928188324
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,7,0.012627199292182922
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,7,0.012622399628162384
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,15,0.01677280068397522
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,15,0.02096319943666458
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,15,0.014824000000953675
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,15,0.014627200365066529
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,15,0.014681600034236908
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,15,0.014718399941921234
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,15,0.014724799990653991
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,15,0.01679839938879013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,15,0.01803999990224838
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,15,0.01387999951839447
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,15,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,15,0.01268479973077774
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,15,0.012771199643611907
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,15,0.012775999307632447
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,31,0.016864000260829924
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,31,0.020852799713611602
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,31,0.01483519971370697
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,31,0.014871999621391296
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,31,0.014718399941921234
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,31,0.014641599357128143
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,31,0.01467680037021637
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,31,0.01690080016851425
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,31,0.01889760047197342
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,31,0.013929599523544311
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,31,0.012801599502563477
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,31,0.012814399600028992
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,31,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,31,0.012608000636100769
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,63,0.016846400499343873
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,63,0.020996800065040587
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,63,0.014769600331783294
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,63,0.014678399264812469
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,63,0.014718399941921234
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,63,0.014644800126552582
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,63,0.01470080018043518
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,63,0.01676799952983856
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,63,0.017926399409770966
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,63,0.01273919939994812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,63,0.012707200646400452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,63,0.013278399407863618
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,63,0.01271039992570877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,63,0.012689599394798278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,127,0.016838400065898894
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,127,0.020956799387931824
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,127,0.014873600006103516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,127,0.014640000462532044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,127,0.014699199795722961
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,127,0.014734399318695069
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,127,0.014769600331783294
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,127,0.016838400065898894
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,127,0.018747200071811677
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,127,0.012782399356365205
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,127,0.01279519945383072
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,127,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,127,0.012641599774360657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,127,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,255,0.019075199961662292
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,255,0.02102559953927994
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,255,0.014846399426460266
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,255,0.014699199795722961
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,255,0.014750400185585022
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,255,0.01467680037021637
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,255,0.014636799693107605
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,255,0.01677280068397522
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,255,0.018680000305175783
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,255,0.012992000579833985
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,255,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,255,0.012699200212955475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,255,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,255,0.013420799374580383
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,511,0.026892799139022826
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,511,0.03141280114650726
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,511,0.018982400000095368
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,511,0.018795199692249298
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,511,0.018787199258804323
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,511,0.018881599605083465
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,511,0.018827199935913086
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,511,0.020183999836444855
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,511,0.02237119972705841
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,511,0.01475519984960556
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,511,0.014686399698257446
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,511,0.014697599411010741
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,511,0.014713600277900696
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,511,0.014732800424098969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,1023,0.049721598625183105
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,1023,0.05586079955101013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,1023,0.04330880045890808
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,1023,0.04141440093517303
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,1023,0.04154399931430817
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,1023,0.04134880006313324
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,1023,0.04136959910392761
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,1023,0.026953598856925963
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,1023,0.031198400259017944
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,1023,0.019016000628471374
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,1023,0.01882079988718033
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,1023,0.01871040016412735
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,1023,0.01866080015897751
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,1023,0.016862399876117706
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,2047,0.07635040283203125
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,2047,0.09580640196800232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,2047,0.06833440065383911
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,2047,0.06434239745140076
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,2047,0.066457599401474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,2047,0.06574400067329407
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,2047,0.0647599995136261
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,2047,0.04587199985980987
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,2047,0.053339201211929324
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,2047,0.0373744010925293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,2047,0.03668479919433594
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,2047,0.03578239977359772
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,2047,0.03648639917373657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,2047,0.03545120060443878
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,4095,0.1338544011116028
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,4095,0.17432479858398436
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,4095,0.11966400146484375
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,4095,0.12252479791641235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,4095,0.1199839949607849
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,4095,0.11927200555801391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,4095,0.11893759965896607
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,4095,0.07241759896278381
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,4095,0.08942880034446717
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,4095,0.05972800254821777
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,4095,0.058462399244308474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,4095,0.0579472005367279
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,4095,0.05787039995193481
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,4095,0.05760480165481567
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,8191,0.23459839820861816
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,8191,0.32644000053405764
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,8191,0.22037279605865479
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,8191,0.21885440349578858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,8191,0.2178015947341919
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,8191,0.21952478885650634
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,8191,0.12028640508651733
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,8191,0.21854400634765625
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,8191,0.15905120372772216
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,8191,0.10494879484176636
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,8191,0.10250879526138305
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,8191,0.1014415979385376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,8191,0.10270400047302246
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,8191,0.10134400129318237
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,1,16383,0.43650240898132325
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,2,16383,0.6270991802215576
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,4,16383,0.4228096008300781
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,8,16383,0.4140431880950928
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,16,16383,0.41423678398132324
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,32,16383,0.41562399864196775
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,128,1,64,16383,0.4182703971862793
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,1,16383,0.21944479942321776
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,2,16383,0.3018671989440918
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,4,16383,0.1913632035255432
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,8,16383,0.19140000343322755
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,16,16383,0.18901599645614625
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,32,16383,0.1889647960662842
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,128,1,64,16383,0.1895151972770691
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,1,0.024982400238513947
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,1,0.031225600838661195
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,1,0.018854400515556334
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,1,0.020083199441432952
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,1,0.020790399610996248
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,1,0.02009280025959015
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,1,0.019550399482250215
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,1,0.02306399941444397
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,1,0.027035200595855714
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,1,0.018931199610233308
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,1,0.016952000558376312
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,1,0.016846400499343873
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,1,0.016872000694274903
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,1,0.016791999340057373
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,3,0.02499839961528778
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,3,0.03134239912033081
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,3,0.020931200683116914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,3,0.01866399943828583
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,3,0.020180800557136537
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,3,0.019952000677585603
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,3,0.020318399369716644
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,3,0.022924800217151643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,3,0.02712480127811432
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,3,0.01881600022315979
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,3,0.016865600645542145
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,3,0.016833600401878358
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,3,0.016787199676036833
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,3,0.016795200109481812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,7,0.025241601467132568
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,7,0.03127360045909881
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,7,0.02096640020608902
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,7,0.020556800067424774
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,7,0.020904000103473663
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,7,0.02070080041885376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,7,0.020815999805927278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,7,0.024558399617671967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,7,0.02709439992904663
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,7,0.018774400651454925
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,7,0.017467199265956877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,7,0.016704000532627106
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,7,0.016755199432373045
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,7,0.01679680049419403
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,15,0.02518239915370941
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,15,0.031142398715019226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,15,0.021035200357437132
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,15,0.020934399962425233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,15,0.02078080028295517
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,15,0.020790399610996248
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,15,0.020945599675178526
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,15,0.02306399941444397
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,15,0.027011200785636902
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,15,0.01889439970254898
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,15,0.01743520051240921
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,15,0.01669120043516159
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,15,0.016764800250530242
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,15,0.01684959977865219
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,31,0.025067201256752013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,31,0.03128960132598877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,31,0.020979200303554536
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,31,0.020899200439453126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,31,0.020956799387931824
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,31,0.02094399929046631
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,31,0.0209184005856514
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,31,0.023078399896621703
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,31,0.027169600129127502
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,31,0.018873600661754607
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,31,0.016812799870967864
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,31,0.016806399822235106
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,31,0.01675039976835251
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,31,0.01671680063009262
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,63,0.025092801451683043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,63,0.03118079900741577
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,63,0.021006399393081666
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,63,0.019828799366950988
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,63,0.020500800013542174
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,63,0.02069920003414154
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,63,0.020945599675178526
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,63,0.026976001262664796
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,63,0.018807999789714813
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,63,0.016923199594020843
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,63,0.023081600666046143
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,63,0.016788800060749055
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,63,0.01674560010433197
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,63,0.01674720048904419
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,127,0.025868800282478333
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,127,0.032734400033950804
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,127,0.02107200026512146
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,127,0.019324800372123717
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,127,0.020870399475097657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,127,0.020870399475097657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,127,0.02091519981622696
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,127,0.023060800135135652
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,127,0.027084800601005554
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,127,0.018811200559139252
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,127,0.016847999393939973
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,127,0.016873599588871004
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,127,0.016777600347995757
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,127,0.0168272003531456
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,255,0.03765920102596283
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,255,0.03733760118484497
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,255,0.022916799783706664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,255,0.020759999752044678
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,255,0.020608000457286835
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,255,0.019871999323368073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,255,0.020820799469947814
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,255,0.027020800113677978
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,255,0.02703840136528015
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,255,0.018862399458885192
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,255,0.01687999963760376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,255,0.01687680035829544
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,255,0.01669919937849045
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,255,0.016780799627304076
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,511,0.05443040132522583
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,511,0.059059202671051025
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,511,0.04415839910507202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,511,0.04152320027351379
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,511,0.041675201058387755
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,511,0.041396799683570865
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,511,0.041457599401474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,511,0.03751519918441772
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,511,0.037364798784255984
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,511,0.02276480048894882
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,511,0.020790399610996248
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,511,0.019449600577354433
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,511,0.020815999805927278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,511,0.02075359970331192
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,1023,0.08382080197334289
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,1023,0.09859679937362671
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,1023,0.07090880274772644
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,1023,0.06800640225410462
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,1023,0.06831520199775695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,1023,0.06687039732933045
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,1023,0.06760479807853699
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,1023,0.05273600220680237
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,1023,0.03975200057029724
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,1023,0.056480002403259275
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,1023,0.037676799297332766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,1023,0.037345600128173825
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,1023,0.03699679970741272
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,1023,0.03766719996929169
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,2047,0.14187840223312378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,2047,0.17634400129318237
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,2047,0.12120800018310547
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,2047,0.1192896008491516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,2047,0.12015039920806884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,2047,0.11668959856033326
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,2047,0.11652640104293824
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,2047,0.08123999834060669
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,2047,0.0957040011882782
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,2047,0.063401597738266
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,2047,0.061236798763275146
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,2047,0.06054239869117737
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,2047,0.06172320246696472
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,4095,0.2520895957946777
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,2047,0.059699201583862306
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,4095,0.332204794883728
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,4095,0.22953920364379882
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,4095,0.2267375946044922
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,4095,0.2228559970855713
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,4095,0.2243648052215576
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,4095,0.2249135971069336
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,4095,0.13151520490646362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,4095,0.16749600172042847
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,4095,0.10961120128631592
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,4095,0.107532799243927
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,4095,0.10510400533676148
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,4095,0.10537760257720948
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,4095,0.10524640083312989
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,1,8191,0.4653295993804932
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,2,8191,0.6285999774932861
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,4,8191,0.42803521156311036
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,8,8191,0.4244991779327393
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,16,8191,0.42532958984375
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,32,8191,0.42950878143310545
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,256,1,64,8191,0.4236303806304932
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,1,8191,0.23117599487304688
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,2,8191,0.3117216110229492
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,4,8191,0.19791200160980224
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,8,8191,0.19481279850006103
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,16,8191,0.1937551975250244
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,32,8191,0.19352960586547852
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,1,0.0435696005821228
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,256,1,64,8191,0.19244639873504638
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,1,0.05176159739494324
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,1,0.03128960132598877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,1,0.029145601391792297
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,1,0.029390400648117064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,1,0.029423999786376952
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,1,0.029260799288749695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,1,0.039113599061965945
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,1,0.043532800674438474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,1,0.027089598774909972
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,1,0.02497279942035675
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,1,0.02295520007610321
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,1,0.023758399486541747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,1,0.023180800676345825
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,3,0.043673598766326906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,3,0.05192480087280273
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,3,0.033283200860023496
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,3,0.03118079900741577
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,3,0.02922239899635315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,3,0.029212799668312073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,3,0.03097440004348755
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,3,0.03946239948272705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,3,0.043412798643112184
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,3,0.027118399739265442
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,3,0.025033599138259886
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,3,0.025216001272201537
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,3,0.022968000173568724
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,3,0.02287680059671402
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,7,0.04405600130558014
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,7,0.051800000667572024
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,7,0.03163839876651764
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,7,0.02940639853477478
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,7,0.030720001459121703
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,7,0.030910399556159974
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,7,0.029524800181388856
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,7,0.03923999965190887
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,7,0.04350239932537079
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,7,0.02712480127811432
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,7,0.022942399978637694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,7,0.02311680018901825
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,7,0.024926400184631346
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,7,0.023257599771022798
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,15,0.04363360106945038
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,15,0.05179679989814758
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,15,0.033185601234436035
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,15,0.031083199381828307
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,15,0.030324798822402955
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,15,0.031204798817634584
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,15,0.029198399186134337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,15,0.039504000544548036
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,15,0.04359999895095825
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,15,0.02720640003681183
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,15,0.024958400428295134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,15,0.02359839975833893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,15,0.022988800704479218
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,15,0.023203200101852416
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,31,0.04525760114192963
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,31,0.052052801847457884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,31,0.031483200192451474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,31,0.031113600730895995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,31,0.029110398888587952
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,31,0.030953601002693176
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,31,0.02909280061721802
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,31,0.03967680037021637
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,31,0.04334560036659241
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,31,0.027241599559783936
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,31,0.024223999679088594
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,31,0.023558400571346283
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,31,0.023056000471115112
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,31,0.023161600530147552
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,63,0.04549759924411774
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,63,0.05379679799079895
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,63,0.033134400844573975
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,63,0.03118560016155243
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,63,0.029257598519325256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,63,0.031142398715019226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,63,0.030313599109649658
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,63,0.04142720103263855
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,63,0.043375998735427856
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,63,0.02722879946231842
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,63,0.02496480047702789
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,63,0.024873599410057068
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,63,0.023057599365711213
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,63,0.023710399866104126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,127,0.049623998999595645
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,127,0.035252800583839415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,127,0.056992000341415404
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,127,0.03124319911003113
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,127,0.030049601197242738
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,127,0.029894399642944335
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,127,0.03123680055141449
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,127,0.04254080057144165
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,127,0.04350399971008301
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,127,0.02715040147304535
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,127,0.02494720071554184
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,127,0.023761600255966187
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,127,0.023057599365711213
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,127,0.0230335995554924
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,255,0.06583359837532043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,255,0.06028159856796265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,255,0.04694719910621643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,255,0.04365920126438141
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,255,0.04293760061264038
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,255,0.04163680076599121
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,255,0.04140639901161194
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,255,0.051708799600601194
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,255,0.028377598524093627
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,255,0.04729120135307312
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,255,0.024505600333213806
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,255,0.025017601251602174
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,255,0.022924800217151643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,255,0.023289600014686586
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,511,0.0957040011882782
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,511,0.09831200242042541
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,511,0.07669119834899903
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,511,0.07079200148582458
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,511,0.07249919772148132
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,511,0.07180320024490357
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,511,0.07030559778213501
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,511,0.0656336009502411
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,511,0.04538399875164032
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,511,0.06236640214920044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,511,0.041257598996162416
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,511,0.03961600065231323
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,511,0.03960480093955994
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,511,0.03940480053424835
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,1023,0.1493343949317932
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,1023,0.167630398273468
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,1023,0.12467999458312988
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,1023,0.12192000150680542
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,1023,0.12064640522003174
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,1023,0.11893279552459717
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,1023,0.1174239993095398
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,1023,0.09460319876670838
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,1023,0.09350079894065857
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,1023,0.06778560280799865
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,1023,0.06460480093955993
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,1023,0.06259520053863525
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,1023,0.06204800009727478
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,1023,0.061844801902770995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,2047,0.2592128038406372
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,2047,0.30581281185150144
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,2047,0.22820160388946534
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,2047,0.22087678909301758
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,2047,0.22063839435577393
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,2047,0.22003519535064697
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,2047,0.22016000747680664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,2047,0.14645440578460694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,2047,0.16254880428314208
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,2047,0.10962879657745361
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,2047,0.11465599536895751
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,2047,0.10912159681320191
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,2047,0.10921599864959716
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,2047,0.10726399421691894
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,1,4095,0.48632001876831055
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,2,4095,0.5742176055908204
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,4,4095,0.4521935939788818
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,8,4095,0.4435743808746338
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,16,4095,0.43276801109313967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,32,4095,0.43928961753845214
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,512,1,64,4095,0.44073920249938964
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,1,4095,0.2508960008621216
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,2,4095,0.287227201461792
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,4,4095,0.2064671993255615
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,8,4095,0.2014672040939331
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,16,4095,0.19831520318984985
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,32,4095,0.1966976046562195
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,512,1,64,4095,0.19639519453048707
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,1,0.07672960162162781
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,1,0.09679520130157471
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,1,0.051836800575256345
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,1,0.049384000897407535
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,1,0.04761120080947876
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,1,0.04783360064029694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,1,0.04749439954757691
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,1,0.0701312005519867
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,1,0.07839199900627136
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,1,0.04542720019817352
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,1,0.03754880130290985
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,1,0.03724960088729858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,1,0.037238401174545285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,1,0.03730719983577728
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,3,0.07668799757957459
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,3,0.0967743992805481
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,3,0.05381280183792114
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,3,0.04962719976902008
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,3,0.04751839935779571
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,3,0.047727999091148374
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,3,0.04770239889621734
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,3,0.07168319821357727
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,3,0.0782368004322052
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,3,0.04530239999294281
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,3,0.039299198985099794
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,3,0.03731360137462616
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,3,0.037134400010108946
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,3,0.03719359934329987
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,7,0.07724800109863281
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,7,0.09701600074768066
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,7,0.0520799994468689
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,7,0.04973280131816864
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,7,0.04782879948616028
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,7,0.04753600060939789
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,7,0.04775680005550385
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,7,0.07077760100364686
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,7,0.07842879891395568
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,7,0.04503200054168701
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,7,0.039156800508499144
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,7,0.037376001477241516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,7,0.03718560039997101
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,7,0.0353520005941391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,15,0.07612159848213196
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,15,0.09747999906539917
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,15,0.05398719906806946
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,15,0.04969600141048432
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,15,0.04748800098896026
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,15,0.04762719869613648
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,15,0.04766559898853302
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,15,0.07139359712600708
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,15,0.07831839919090271
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,15,0.04547359943389893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,15,0.039297598600387576
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,15,0.037363201379776
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,15,0.037257599830627444
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,15,0.035729598999023435
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,31,0.07671200037002564
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,31,0.09680320024490356
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,31,0.05536320209503174
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,31,0.049579200148582456
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,31,0.04878239929676056
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,31,0.04756479859352112
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,31,0.07202240228652954
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,31,0.08033279776573181
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,31,0.04751519858837128
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,31,0.045633599162101746
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,31,0.03918560147285462
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,31,0.03731040060520172
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,31,0.03723360002040863
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,31,0.03522560000419617
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,63,0.07852640151977539
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,63,0.09711199998855591
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,63,0.05779359936714172
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,63,0.04990079998970032
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,63,0.04935519993305206
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,63,0.04900160133838653
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,63,0.04778240025043488
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,63,0.07244319915771484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,63,0.08049600124359131
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,63,0.04564479887485504
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,63,0.03938559889793396
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,63,0.03728959858417511
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,63,0.03733119964599609
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,127,0.0867904007434845
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,63,0.0372512012720108
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,127,0.09874719977378846
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,127,0.06474720239639283
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,127,0.05773599743843079
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,127,0.05583999752998352
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,127,0.05440160036087036
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,127,0.05507040023803711
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,127,0.07245759963989258
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,127,0.08099679946899414
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,127,0.048785600066185
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,127,0.03946239948272705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,127,0.03792639970779419
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,127,0.03732799887657166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,127,0.03569439947605133
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,255,0.11357439756393432
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,255,0.10780800580978393
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,255,0.07724639773368835
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,255,0.07024319767951966
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,255,0.06880800127983093
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,255,0.06777439713478088
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,255,0.06769599914550781
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,255,0.08498240113258362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,255,0.08004159927368164
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,255,0.05098559856414795
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,255,0.0446832001209259
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,255,0.042737600207328794
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,255,0.04284160137176514
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,255,0.0416159987449646
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,511,0.16341279745101928
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,511,0.17829920053482057
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,511,0.1313599944114685
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,511,0.12184799909591675
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,511,0.11949759721755981
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,511,0.11898560523986816
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,511,0.12114399671554565
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,511,0.1091088056564331
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,511,0.11007200479507447
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,511,0.06659200191497802
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,511,0.07242879867553711
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,511,0.06352800130844116
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,511,0.06255679726600646
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,511,0.06222879886627197
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,1023,0.2639568090438843
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,1023,0.3204832077026367
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,1023,0.22107999324798583
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,1023,0.21396160125732422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,1023,0.20962240695953369
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,1023,0.2055039882659912
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,1023,0.20604801177978516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,1023,0.15903520584106445
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,1023,0.11679359674453735
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,1023,0.17460160255432128
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,1023,0.10904480218887329
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,1023,0.10548640489578247
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,1023,0.1047711968421936
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,1023,0.10318559408187866
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,1,2047,0.46444478034973147
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,2,2047,0.5950191974639892
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,4,2047,0.410265588760376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,8,2047,0.39670720100402834
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,16,2047,0.3975104093551636
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,32,2047,0.3930720090866089
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,1,1024,1,64,2047,0.3927567958831787
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,1,2047,0.25984959602355956
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,2,2047,0.30608320236206055
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,4,2047,0.20211200714111327
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,8,2047,0.1946992039680481
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,16,2047,0.19053759574890136
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,32,2047,0.18914719820022582
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,1,1024,1,64,2047,0.18757760524749756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,1,0.012715199589729309
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,1,0.01268640011548996
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,1,0.012670400738716125
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,1,0.010684800148010255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,1,0.011062400043010711
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,1,0.010702399909496308
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,1,0.010745599865913391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,1,0.010417599976062775
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,1,0.010756800323724747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,1,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,1,0.010751999914646149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,1,0.010577599704265594
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,1,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,1,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,3,0.012620800733566284
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,3,0.012691199779510498
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,3,0.012644800543785095
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,3,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,3,0.010657600313425063
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,3,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,3,0.010756800323724747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,3,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,3,0.010836800187826156
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,3,0.010769599676132202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,3,0.010751999914646149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,3,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,3,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,3,0.010664000362157821
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,7,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,7,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,7,0.012617599964141846
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,7,0.01080320030450821
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,7,0.010873600095510482
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,7,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,7,0.01072319969534874
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,7,0.010702399909496308
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,7,0.010795199871063232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,7,0.010523200035095215
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,7,0.010769599676132202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,7,0.010683199763298035
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,7,0.010812799632549285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,7,0.010593599826097488
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,15,0.012588800489902496
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,15,0.012336000055074691
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,15,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,15,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,15,0.010763200372457505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,15,0.010715200006961823
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,15,0.010683199763298035
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,15,0.010681600123643876
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,15,0.01075040027499199
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,15,0.01075040027499199
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,15,0.010822399705648422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,15,0.010628800094127654
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,15,0.010831999778747558
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,15,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,31,0.012521600723266602
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,31,0.012675200402736665
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,31,0.012656000256538392
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,31,0.01072319969534874
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,31,0.010740800201892853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,31,0.010627199709415436
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,31,0.010779199749231338
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,31,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,31,0.01088000014424324
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,31,0.010655999928712846
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,31,0.010806400328874588
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,31,0.010499200224876404
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,31,0.01077599972486496
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,31,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,63,0.012694400548934937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,63,0.012611199915409089
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,63,0.012664000689983367
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,63,0.010700800269842149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,63,0.010886400192975997
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,63,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,63,0.010732799768447876
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,63,0.010708799958229065
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,63,0.01090560033917427
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,63,0.01075040027499199
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,63,0.010599999874830245
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,63,0.010716799646615982
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,63,0.010623999685049058
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,63,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,127,0.012868799269199371
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,127,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,127,0.012699200212955475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,127,0.010849600285291671
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,127,0.010691200196743012
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,127,0.010830400139093399
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,127,0.010683199763298035
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,127,0.011483199894428253
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,127,0.010791999846696853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,127,0.010753600299358368
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,127,0.010555200278759003
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,127,0.010828799754381179
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,127,0.010679999738931656
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,127,0.010664000362157821
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,255,0.010811199992895126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,255,0.012724800407886505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,255,0.011684799939393998
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,255,0.010913600027561188
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,255,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,255,0.010768000036478043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,255,0.010571199655532836
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,255,0.010763200372457505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,255,0.01061279997229576
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,255,0.01079839989542961
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,255,0.010635200142860412
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,255,0.010833600163459777
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,255,0.010625600069761276
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,255,0.010796800255775452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,511,0.01464959979057312
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,511,0.014601600170135499
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,511,0.014603200554847717
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,511,0.013060800731182098
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,511,0.012647999823093415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,511,0.01273919939994812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,511,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,511,0.012675200402736665
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,511,0.01265760064125061
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,511,0.012705600261688233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,511,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,511,0.012608000636100769
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,511,0.01255200058221817
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,511,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,1023,0.016734400391578676
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,1023,0.016956800222396852
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,1023,0.016795200109481812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,1023,0.016711999475955964
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,1023,0.014771200716495514
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,1023,0.014679999649524688
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,1023,0.014886400103569031
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,1023,0.014881600439548493
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,1023,0.014767999947071075
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,1023,0.01467359960079193
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,1023,0.014824000000953675
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,1023,0.013513599336147309
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,1023,0.012732799351215362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,1023,0.012894399464130402
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,2047,0.01698720008134842
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,2047,0.01693120002746582
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,2047,0.01682399958372116
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,2047,0.014998400211334228
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,2047,0.01472959965467453
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,2047,0.014886400103569031
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,2047,0.014726400375366211
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,2047,0.014812800288200378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,2047,0.01483519971370697
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,2047,0.014878399670124054
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,2047,0.014720000326633453
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,2047,0.012862400710582733
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,2047,0.013358399271965027
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,2047,0.014705599844455719
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,4095,0.021014399826526642
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,4095,0.017689600586891174
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,4095,0.01701119989156723
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,4095,0.016795200109481812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,4095,0.014822399616241455
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,4095,0.014791999757289887
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,4095,0.014971199631690978
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,4095,0.01884479969739914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,4095,0.01674399971961975
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,4095,0.014776000380516052
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,4095,0.014883199334144592
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,4095,0.014627200365066529
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,4095,0.014679999649524688
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,4095,0.014660799503326416
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,8191,0.02704319953918457
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,8191,0.02486560046672821
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,8191,0.018905599415302277
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,8191,0.01714719980955124
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,8191,0.01687040030956268
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,8191,0.01688639968633652
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,8191,0.016864000260829924
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,8191,0.020873600244522096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,8191,0.019463999569416045
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,8191,0.01749439984560013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,8191,0.01685599982738495
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,8191,0.014990399777889251
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,8191,0.016739200055599212
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,16383,0.037887999415397645
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,16383,0.029156801104545594
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,8191,0.01674239933490753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,16383,0.027136000990867614
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,16383,0.02099840044975281
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,16383,0.020771199464797975
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,16383,0.019420799612998963
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,16383,0.02083200067281723
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,16383,0.027108800411224366
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,16383,0.022988800704479218
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,16383,0.021012799441814424
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,16383,0.019016000628471374
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,16383,0.018852800130844116
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,16383,0.018831999599933626
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,16383,0.018803200125694274
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,32767,0.03305279910564422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,32767,0.03765600025653839
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,32767,0.031281599402427675
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,32767,0.02903839945793152
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,32767,0.02858400046825409
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,32767,0.02717440128326416
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,32767,0.027599999308586122
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,32767,0.029624000191688538
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,32767,0.02717759907245636
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,32767,0.024900799989700316
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,32767,0.025094398856163026
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,32767,0.02298240065574646
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,32767,0.02301120012998581
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,32767,0.02290239930152893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,65535,0.03746399879455566
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,65535,0.03952800035476685
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,65535,0.04430879950523377
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,65535,0.03672159910202026
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,65535,0.03519040048122406
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,65535,0.03419840037822723
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,65535,0.03788639903068543
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,65535,0.03314560055732727
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,65535,0.034457600116729735
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,65535,0.029049599170684816
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,65535,0.029110398888587952
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,65535,0.02871359884738922
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,65535,0.029158401489257812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,65535,0.02906079888343811
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1,1,1,131071,0.05954880118370056
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1,1,2,131071,0.06207680106163025
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1,1,4,131071,0.07342720031738281
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1,1,8,131071,0.05410879850387573
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1,1,16,131071,0.05384320020675659
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1,1,32,131071,0.0557919979095459
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1,1,64,131071,0.0554032027721405
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1,1,1,131071,0.04158399999141693
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1,1,2,131071,0.04038400053977966
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1,1,8,131071,0.034867200255393985
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1,1,16,131071,0.03337279856204987
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1,1,4,131071,0.042396798729896545
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1,1,32,131071,0.034750398993492124
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1,1,64,131071,0.03315359950065613
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,1,0.012614400684833526
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,1,0.011793600022792816
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,1,0.012580800056457519
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,1,0.012492799758911132
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,1,0.010766399651765823
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,1,0.010683199763298035
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,1,0.010819199681282043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,1,0.010761599987745285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,1,0.010860799998044967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,1,0.010838399827480315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,1,0.010811199992895126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,1,0.010737600177526474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,1,0.010710400342941285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,1,0.01071999967098236
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,3,0.012598399817943574
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,3,0.0126351997256279
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,3,0.012734399735927581
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,3,0.012745599448680877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,3,0.010817600041627884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,3,0.01135680004954338
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,3,0.010771200060844421
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,3,0.010734400153160096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,3,0.010758399963378906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,3,0.010780800133943558
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,3,0.010841599851846694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,3,0.010601600259542465
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,3,0.010556799918413162
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,3,0.010596799850463866
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,7,0.012761600315570831
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,7,0.01096159964799881
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,7,0.012619200348854064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,7,0.012529599666595458
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,7,0.010918399691581726
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,7,0.010945600271224976
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,7,0.010744000226259232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,7,0.010673599690198899
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,7,0.011523199826478958
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,7,0.010694400221109391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,7,0.010836800187826156
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,7,0.010652799904346467
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,7,0.010604800283908844
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,7,0.010883200168609618
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,15,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,15,0.012801599502563477
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,15,0.012561599910259246
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,15,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,15,0.010815999656915664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,15,0.010784000158309937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,15,0.010779199749231338
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,15,0.011428800225257874
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,15,0.010806400328874588
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,15,0.010787200182676315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,15,0.010870400071144103
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,15,0.010728000104427338
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,15,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,15,0.01091040000319481
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,31,0.012673600018024445
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,31,0.01255359947681427
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,31,0.012643200159072877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,31,0.012619200348854064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,31,0.010911999642848969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,31,0.010819199681282043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,31,0.010708799958229065
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,31,0.010768000036478043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,31,0.010718400031328202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,31,0.011233600229024887
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,31,0.010684800148010255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,31,0.010651200264692306
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,31,0.010977599769830704
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,31,0.010831999778747558
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,63,0.012702399492263794
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,63,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,63,0.012051200121641159
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,63,0.012751999497413635
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,63,0.010689599812030793
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,63,0.010791999846696853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,63,0.01069760024547577
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,63,0.010876800119876861
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,63,0.010791999846696853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,63,0.010892800241708755
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,63,0.010679999738931656
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,63,0.010630399733781815
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,63,0.010595200210809707
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,63,0.01067200005054474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,127,0.01271360069513321
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,127,0.012593600153923034
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,127,0.012585599720478059
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,127,0.010788799822330475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,127,0.01276639997959137
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,127,0.010862399637699128
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,127,0.010489600151777268
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,127,0.010787200182676315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,127,0.010785599797964096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,127,0.010716799646615982
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,127,0.01067039966583252
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,127,0.010622400045394897
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,127,0.010814400017261505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,127,0.010715200006961823
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,255,0.012680000066757202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,255,0.012608000636100769
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,255,0.01271360069513321
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,255,0.012518399953842163
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,255,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,255,0.01067200005054474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,255,0.010862399637699128
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,255,0.01162559986114502
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,255,0.010620799660682679
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,255,0.011107199639081956
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,255,0.010766399651765823
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,255,0.010683199763298035
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,255,0.01072160005569458
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,255,0.010665600001811982
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,511,0.014684799313545226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,511,0.014742399752140044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,511,0.014803199470043183
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,511,0.01467359960079193
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,511,0.012950399518013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,511,0.01284639984369278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,511,0.01247360035777092
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,511,0.012700800597667695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,511,0.012673600018024445
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,511,0.012664000689983367
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,511,0.012555199861526489
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,511,0.012567999958992004
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,511,0.012582400441169738
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,511,0.012807999551296235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,1023,0.016806399822235106
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,1023,0.014812800288200378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,1023,0.014867199957370758
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,1023,0.014796799421310425
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,1023,0.0147024005651474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,1023,0.012726399302482604
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,1023,0.012928000092506409
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,1023,0.014744000136852264
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,1023,0.012775999307632447
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,1023,0.01276639997959137
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,1023,0.012851199507713318
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,1023,0.012750400602817536
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,1023,0.012639999389648438
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,1023,0.012670400738716125
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,2047,0.01759520024061203
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,2047,0.016788800060749055
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,2047,0.014791999757289887
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,2047,0.014849600195884705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,2047,0.014659200608730317
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,2047,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,2047,0.012787200510501862
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,2047,0.014977599680423736
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,2047,0.014828799664974213
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,2047,0.012750400602817536
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,2047,0.012744000554084778
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,2047,0.012849600613117218
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,2047,0.012836800515651703
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,2047,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,4095,0.020851199328899384
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,4095,0.01897760033607483
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,4095,0.01884319931268692
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,4095,0.014742399752140044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,4095,0.014787200093269347
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,4095,0.01467680037021637
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,4095,0.014697599411010741
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,4095,0.016812799870967864
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,4095,0.016756799817085267
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,4095,0.016833600401878358
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,4095,0.012870399653911591
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,4095,0.01279360055923462
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,4095,0.01263359934091568
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,4095,0.012942400574684144
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,8191,0.031064000725746155
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,8191,0.020934399962425233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,8191,0.020787200331687926
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,8191,0.01897920072078705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,8191,0.018796800076961516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,8191,0.016814400255680085
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,8191,0.017796799540519714
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,8191,0.023060800135135652
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,8191,0.018772800266742707
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,8191,0.016760000586509706
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,8191,0.016921600699424742
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,8191,0.01680160015821457
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,8191,0.016734400391578676
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,16383,0.027024000883102417
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,8191,0.016648000478744505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,16383,0.03333120048046112
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,16383,0.02503040134906769
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,16383,0.022859199345111846
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,16383,0.02093919962644577
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,16383,0.02196960002183914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,16383,0.020846399664878845
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,16383,0.024959999322891235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,16383,0.02492479979991913
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,16383,0.02084160000085831
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,16383,0.019064000248908995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,16383,0.018822400271892546
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,16383,0.018795199692249298
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,16383,0.018811200559139252
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,32767,0.03213280141353607
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,32767,0.037257599830627444
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,32767,0.03615039885044098
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,32767,0.02906239926815033
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,32767,0.02714880108833313
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,32767,0.027033600211143493
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,32767,0.026236799359321595
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,32767,0.02730720043182373
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,32767,0.039105600118637084
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,32767,0.02688480019569397
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,32767,0.02335679978132248
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,32767,0.023209600150585173
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,32767,0.024916799366474153
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,32767,0.022976000607013703
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,65535,0.051819199323654176
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,65535,0.05876320004463196
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,65535,0.057815998792648315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,65535,0.05376960039138794
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,65535,0.05314080119132995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,65535,0.054504001140594484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,65535,0.05448319911956787
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,65535,0.03538079857826233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,65535,0.03929120004177093
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,65535,0.03402239978313446
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,65535,0.031323200464248656
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,65535,0.031193599104881287
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,65535,0.030619201064109803
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,65535,0.031150400638580322
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,2,1,1,131071,0.08072320222854615
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,2,1,2,131071,0.08528159856796265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,2,1,4,131071,0.09241440296173095
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,2,1,8,131071,0.07841759920120239
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,2,1,16,131071,0.076883202791214
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,2,1,32,131071,0.0781503975391388
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,2,1,64,131071,0.07865279912948608
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,2,1,1,131071,0.053574401140213015
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,2,1,2,131071,0.056118398904800415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,2,1,4,131071,0.0534991979598999
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,2,1,8,131071,0.047547200322151185
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,2,1,16,131071,0.04583359956741333
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,2,1,32,131071,0.049028798937797546
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,2,1,64,131071,0.04821920096874237
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,1,0.012572799623012543
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,1,0.012579199671745301
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,1,0.012854400277137756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,1,0.012601600587368011
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,1,0.011884800344705581
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,1,0.010844799876213073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,1,0.010700800269842149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,1,0.01263200044631958
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,1,0.01085439994931221
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,1,0.010815999656915664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,1,0.010785599797964096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,1,0.010713600367307664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,1,0.010844799876213073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,1,0.010740800201892853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,3,0.012734399735927581
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,3,0.012729600071907043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,3,0.01268640011548996
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,3,0.012700800597667695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,3,0.010705599933862687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,3,0.01069599986076355
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,3,0.011598400026559829
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,3,0.012703999876976013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,3,0.01063840016722679
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,3,0.010708799958229065
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,3,0.01075040027499199
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,3,0.010763200372457505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,3,0.010598400235176086
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,7,0.012891200184822083
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,3,0.010524799674749374
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,7,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,7,0.012881599366664886
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,7,0.010804799944162368
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,7,0.01268479973077774
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,7,0.0107744000852108
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,7,0.01064639985561371
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,7,0.01271200031042099
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,7,0.012137600034475327
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,7,0.010768000036478043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,7,0.010648000240325927
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,7,0.010843200236558914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,7,0.010820800065994262
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,7,0.010764800012111664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,15,0.012860800325870513
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,15,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,15,0.012691199779510498
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,15,0.012822400033473968
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,15,0.012060800194740295
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,15,0.01104160025715828
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,15,0.010952000319957734
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,15,0.012895999848842621
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,15,0.010745599865913391
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,15,0.010900799930095673
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,15,0.010891199856996537
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,15,0.010726399719715118
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,15,0.010795199871063232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,15,0.01072480008006096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,31,0.012828800082206725
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,31,0.012619200348854064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,31,0.012680000066757202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,31,0.012676799297332763
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,31,0.011822400242090225
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,31,0.010760000348091126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,31,0.01077279970049858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,31,0.012646399438381195
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,31,0.01088000014424324
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,31,0.010753600299358368
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,31,0.010716799646615982
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,31,0.010655999928712846
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,31,0.010817600041627884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,31,0.010606399923563003
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,63,0.012782399356365205
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,63,0.012646399438381195
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,63,0.012652799487113953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,63,0.012646399438381195
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,63,0.010849600285291671
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,63,0.010814400017261505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,63,0.01085280030965805
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,63,0.01276959925889969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,63,0.010476800054311753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,63,0.01069760024547577
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,63,0.0108815997838974
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,63,0.010763200372457505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,63,0.01066880002617836
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,63,0.010744000226259232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,127,0.012848000228404998
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,127,0.012667199969291687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,127,0.012721599638462066
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,127,0.012785600125789642
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,127,0.012598399817943574
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,127,0.010707200318574906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,127,0.010902400314807891
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,127,0.012683199346065521
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,127,0.012385600060224534
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,127,0.010856000334024429
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,127,0.010711999982595444
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,127,0.0106175996363163
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,127,0.010751999914646149
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,127,0.011323200166225433
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,255,0.012990400195121765
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,255,0.01266240030527115
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,255,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,255,0.012753599882125854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,255,0.012460800260305405
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,255,0.010814400017261505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,255,0.010785599797964096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,255,0.012556800246238708
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,255,0.010817600041627884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,255,0.01088479980826378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,255,0.010777600109577179
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,255,0.010760000348091126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,255,0.01075040027499199
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,255,0.010924799740314484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,511,0.014923200011253357
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,511,0.014742399752140044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,511,0.014707200229167938
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,511,0.014697599411010741
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,511,0.013926400244235993
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,511,0.012958399951457977
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,511,0.012761600315570831
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,511,0.012768000364303589
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,511,0.012839999794960023
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,511,0.012638400495052337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,511,0.0127920001745224
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,511,0.012636800110340119
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,511,0.01271200031042099
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,511,0.011655999720096588
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,1023,0.016923199594020843
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,1023,0.016836799681186676
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,1023,0.014883199334144592
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,1023,0.014596800506114959
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,1023,0.014819200336933135
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,1023,0.01467519998550415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,1023,0.014777599275112152
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,1023,0.014774399995803832
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,1023,0.014772799611091614
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,1023,0.01289760023355484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,1023,0.012993599474430084
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,1023,0.01276639997959137
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,1023,0.012715199589729309
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,1023,0.012835200130939483
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,2047,0.020875200629234314
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,2047,0.017294399440288544
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,2047,0.0169855996966362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,2047,0.014865599572658539
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,2047,0.014884799718856812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,2047,0.014684799313545226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,2047,0.013872000575065612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,2047,0.0168272003531456
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,2047,0.014945599436759948
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,2047,0.014750400185585022
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,2047,0.012982399761676788
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,2047,0.012923200428485871
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,2047,0.012895999848842621
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,2047,0.012590399384498597
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,4095,0.028678399324417115
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,4095,0.020846399664878845
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,4095,0.019332799315452575
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,4095,0.018838399648666383
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,4095,0.017769600450992584
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,4095,0.016945600509643555
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,4095,0.021113599836826324
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,4095,0.01706559956073761
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,4095,0.016991999745368958
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,4095,0.017136000096797943
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,4095,0.01712000072002411
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,4095,0.01515199989080429
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,4095,0.015676799416542053
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,4095,0.015300799906253815
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,8191,0.02345760017633438
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,8191,0.031240001320838928
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,8191,0.022752000391483305
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,8191,0.021331200003623964
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,8191,0.01944800019264221
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,8191,0.01911199986934662
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,8191,0.022270399332046508
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,8191,0.019382399320602418
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,8191,0.02330559939146042
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,8191,0.01921759992837906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,8191,0.017243200540542604
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,8191,0.01698880046606064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,8191,0.018119999766349794
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,8191,0.017103999853134155
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,16383,0.02954080104827881
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,16383,0.03171359896659851
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,16383,0.034720000624656674
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,16383,0.02526400089263916
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,16383,0.02377600073814392
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,16383,0.023503999412059783
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,16383,0.023579199612140656
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,16383,0.025488001108169556
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,16383,0.03151200115680695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,16383,0.02378239929676056
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,16383,0.020078399777412416
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,16383,0.021403199434280394
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,16383,0.021241599321365358
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,16383,0.0192208006978035
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,32767,0.04872959852218628
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,32767,0.0539247989654541
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,32767,0.055340802669525145
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,32767,0.04955039918422699
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,32767,0.0493120014667511
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,32767,0.05004159808158874
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,32767,0.04920479953289032
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,32767,0.0330159991979599
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,32767,0.03580160140991211
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,32767,0.031999999284744264
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,32767,0.02749119997024536
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,32767,0.027350398898124694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,32767,0.027507200837135315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,32767,0.025300800800323486
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,65535,0.07602880001068116
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,65535,0.07954879999160766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,65535,0.07355200052261353
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,65535,0.09117280244827271
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,65535,0.07201759815216065
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,65535,0.07268959879875184
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,65535,0.0729632019996643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,65535,0.04996800124645233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,65535,0.05087199807167053
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,65535,0.05088000297546387
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,65535,0.04406079947948456
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,65535,0.041998401284217834
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,65535,0.0434224009513855
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,65535,0.043208000063896176
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,4,1,1,131071,0.12882399559020996
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,4,1,2,131071,0.13008320331573486
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,4,1,4,131071,0.15976799726486207
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,4,1,8,131071,0.12411199808120728
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,4,1,16,131071,0.12375520467758179
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,4,1,32,131071,0.12229599952697753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,4,1,64,131071,0.12339040040969848
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,4,1,1,131071,0.08028960227966309
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,4,1,2,131071,0.07693759799003601
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,4,1,4,131071,0.08414559960365295
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,4,1,8,131071,0.06554719805717468
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,4,1,16,131071,0.06365280151367188
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,4,1,32,131071,0.06466400027275085
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,4,1,64,131071,0.06393920183181763
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,1,0.013259199261665345
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,1,0.01337919980287552
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,1,0.01310880035161972
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,1,0.012830400466918945
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,1,0.012108799815177918
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,1,0.013043199479579926
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,1,0.011764799803495407
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,1,0.012923200428485871
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,1,0.012753599882125854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,1,0.011046399921178817
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,1,0.0117296002805233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,1,0.0110384002327919
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,1,0.01104160025715828
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,1,0.011147200316190719
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,3,0.012932799756526947
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,3,0.013891200721263885
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,3,0.012971200048923492
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,3,0.012943999469280243
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,3,0.013031999766826629
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,3,0.013228799402713775
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,3,0.01122559979557991
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,3,0.013260799646377563
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,3,0.01329440027475357
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,3,0.013899199664592743
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,3,0.011182399839162827
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,3,0.01120320037007332
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,3,0.01053439974784851
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,3,0.012280000001192093
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,7,0.01326719969511032
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,7,0.013227200508117676
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,7,0.013094399869441987
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,7,0.010956799983978272
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,7,0.01308320015668869
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,7,0.012995199859142303
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,7,0.011121600121259689
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,7,0.013214400410652161
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,7,0.013084800541400909
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,7,0.012507200241088867
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,7,0.011217600107192994
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,7,0.010903999954462052
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,7,0.011400000005960465
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,7,0.011217600107192994
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,15,0.01329279989004135
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,15,0.013195200264453888
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,15,0.013259199261665345
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,15,0.013006399571895599
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,15,0.012432000041007996
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,15,0.011771199852228164
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,15,0.013096000254154205
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,15,0.01141119971871376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,15,0.013035200536251068
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,15,0.011159999668598175
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,15,0.011470399796962738
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,15,0.011383999884128571
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,15,0.010807999968528747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,15,0.011132799834012986
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,31,0.013545599579811097
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,31,0.012996800243854523
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,31,0.013438400626182557
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,31,0.012947200238704682
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,31,0.012745599448680877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,31,0.012982399761676788
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,31,0.01133119985461235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,31,0.013064000010490417
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,31,0.012895999848842621
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,31,0.011748799681663513
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,31,0.011241599917411804
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,31,0.011259199678897857
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,31,0.011300799995660782
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,63,0.013443200290203095
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,31,0.01199679970741272
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,63,0.013174399733543396
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,63,0.013052800297737121
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,63,0.013016000390052795
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,63,0.013097600638866424
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,63,0.013035200536251068
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,63,0.01112319976091385
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,63,0.013009600341320038
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,63,0.013209599256515502
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,63,0.012371200323104858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,63,0.01122559979557991
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,63,0.011076799780130386
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,63,0.011631999909877778
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,63,0.01117279976606369
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,127,0.013206399977207184
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,127,0.013259199261665345
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,127,0.012934400141239167
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,127,0.013196800649166108
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,127,0.011159999668598175
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,127,0.012980799376964568
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,127,0.011345600336790084
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,127,0.013019199669361114
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,127,0.012967999279499053
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,127,0.011984000355005265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,127,0.011240000277757645
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,127,0.0110944002866745
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,127,0.011473599821329117
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,127,0.011030399799346923
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,255,0.014734399318695069
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,255,0.013335999846458436
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,255,0.013099199533462525
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,255,0.012982399761676788
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,255,0.013422399759292603
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,255,0.012347199767827988
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,255,0.012707200646400452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,255,0.012203200161457062
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,255,0.01321759968996048
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,255,0.01120000034570694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,255,0.0110895998775959
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,255,0.011459200084209442
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,255,0.01112319976091385
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,255,0.011057599633932113
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,511,0.01706559956073761
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,511,0.015374399721622467
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,511,0.015136000514030457
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,511,0.015052799880504609
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,511,0.015091200172901154
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,511,0.01510079950094223
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,511,0.01329759955406189
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,511,0.015038399398326874
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,511,0.01313920021057129
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,511,0.01390880048274994
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,511,0.013051199913024902
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,511,0.013016000390052795
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,511,0.012984000146389008
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,1023,0.02125599980354309
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,511,0.013222399353981017
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,1023,0.017260800302028655
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,1023,0.01714559942483902
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,1023,0.015273599326610566
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,1023,0.01541599929332733
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,1023,0.015299199521541596
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,1023,0.015062400698661804
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,1023,0.01721920073032379
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,1023,0.015462400019168853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,1023,0.01536639928817749
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,1023,0.01321759968996048
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,1023,0.013199999928474426
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,1023,0.013102400302886962
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,2047,0.02789280116558075
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,1023,0.013289600610733032
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,2047,0.021385599672794343
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,2047,0.019144000113010408
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,2047,0.017294399440288544
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,2047,0.017425599694252013
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,2047,0.01732800006866455
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,2047,0.01700640022754669
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,2047,0.02146719992160797
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,2047,0.017715199291706084
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,2047,0.01536639928817749
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,2047,0.015054400265216827
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,2047,0.014907200634479523
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,2047,0.014772799611091614
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,2047,0.015184000134468079
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,4095,0.02937920093536377
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,4095,0.02335679978132248
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,4095,0.021460799872875212
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,4095,0.019326399266719817
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,4095,0.019225600361824035
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,4095,0.01900479942560196
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,4095,0.017350399494171144
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,4095,0.021507200598716737
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,4095,0.021214400231838227
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,4095,0.01732639968395233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,4095,0.01733119934797287
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,4095,0.017056000232696534
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,4095,0.015292799472808838
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,4095,0.015292799472808838
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,8191,0.028649601340293884
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,8191,0.03349120020866394
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,8191,0.02985759973526001
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,8191,0.023307199776172637
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,8191,0.021748800575733186
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,8191,0.021483199298381807
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,8191,0.021697600185871125
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,8191,0.025672000646591187
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,8191,0.02553919851779938
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,8191,0.0232464000582695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,8191,0.019545599818229675
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,8191,0.019278399646282196
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,8191,0.019121600687503813
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,8191,0.017343999445438386
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,16383,0.049297600984573364
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,16383,0.05121600031852722
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,16383,0.056846398115158084
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,16383,0.047307199239730834
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,16383,0.0477728009223938
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,16383,0.04643999934196472
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,16383,0.04599359929561615
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,16383,0.03252480030059814
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,16383,0.029756799340248108
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,16383,0.03152639865875244
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,16383,0.025235199928283693
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,16383,0.024222399294376373
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,16383,0.023984000086784363
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,16383,0.02324160039424896
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,32767,0.07556800246238708
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,32767,0.07580159902572632
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,32767,0.06977120041847229
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,32767,0.09411839842796325
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,32767,0.06949599981307983
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,32767,0.06799839735031128
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,32767,0.07045279741287232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,32767,0.05052000284194946
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,32767,0.0467631995677948
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,32767,0.052300798892974856
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,32767,0.041678398847579956
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,32767,0.040833601355552675
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,32767,0.03984479904174805
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,32767,0.03967039883136749
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,65535,0.12963199615478516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,65535,0.1302832007408142
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,65535,0.16331520080566406
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,65535,0.1208799958229065
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,65535,0.12056959867477417
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,65535,0.12083200216293336
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,65535,0.12046719789505005
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,65535,0.07863680124282837
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,65535,0.07290080189704895
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,65535,0.08543040156364441
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,65535,0.06084799766540527
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,65535,0.062459200620651245
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,65535,0.06130880117416382
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,65535,0.060577601194381714
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,8,1,1,131071,0.23029119968414308
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,8,1,2,131071,0.2340991973876953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,8,1,4,131071,0.3035343885421753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,8,1,8,131071,0.21822080612182618
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,8,1,16,131071,0.21641120910644532
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,8,1,32,131071,0.21705279350280762
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,8,1,64,131071,0.2175503969192505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,8,1,1,131071,0.13323999643325807
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,8,1,2,131071,0.11799039840698242
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,8,1,4,131071,0.14943840503692626
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,8,1,8,131071,0.10679359436035156
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,8,1,16,131071,0.10482399463653565
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,8,1,32,131071,0.10392960309982299
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,8,1,64,131071,0.10404000282287598
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,1,0.015190400183200836
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,1,0.01321599930524826
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,1,0.013156799972057343
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,1,0.01289760023355484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,1,0.013099199533462525
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,1,0.011134400218725204
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,1,0.012887999415397644
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,1,0.015059199929237366
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,1,0.013340799510478974
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,1,0.013044799864292144
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,1,0.011215999722480774
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,1,0.011392000317573547
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,1,0.011049599945545196
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,1,0.011419200152158738
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,3,0.015167999267578124
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,3,0.01316480040550232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,3,0.013179199397563934
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,3,0.012990400195121765
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,3,0.013465599715709686
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,3,0.012940800189971924
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,3,0.01289760023355484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,3,0.014734399318695069
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,3,0.013209599256515502
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,3,0.01302880048751831
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,3,0.010972800105810166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,3,0.011158400028944016
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,3,0.011448000371456147
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,7,0.015087999403476715
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,3,0.01096320003271103
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,7,0.013283200562000275
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,7,0.013211199641227722
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,7,0.012824000418186187
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,7,0.012615999579429627
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,7,0.013055999577045441
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,7,0.013038399815559387
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,7,0.014020800590515137
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,7,0.013040000200271606
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,7,0.013592000305652618
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,7,0.012699200212955475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,7,0.011102399975061416
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,7,0.01112319976091385
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,7,0.01186240017414093
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,15,0.015048000216484069
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,15,0.013096000254154205
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,15,0.013059200346469879
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,15,0.013153600692749023
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,15,0.013027200102806091
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,15,0.01284479945898056
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,15,0.012828800082206725
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,15,0.012807999551296235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,15,0.012863999605178833
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,15,0.01424960047006607
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,15,0.010958400368690491
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,15,0.011059200018644333
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,15,0.010838399827480315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,31,0.015112000703811645
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,15,0.011358399689197541
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,31,0.01300159990787506
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,31,0.01305759996175766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,31,0.01316480040550232
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,31,0.01326719969511032
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,31,0.012680000066757202
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,31,0.012995199859142303
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,31,0.013284799456596375
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,31,0.013031999766826629
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,31,0.012996800243854523
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,31,0.011966399848461151
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,31,0.01098719984292984
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,31,0.0114656001329422
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,31,0.011084800213575363
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,63,0.014744000136852264
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,63,0.013166399300098419
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,63,0.014151999354362487
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,63,0.012988799810409546
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,63,0.012995199859142303
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,63,0.012884800136089326
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,63,0.012971200048923492
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,63,0.013118399679660797
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,63,0.013115200400352477
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,63,0.012887999415397644
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,63,0.01247360035777092
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,63,0.011003199964761734
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,63,0.011180800199508668
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,63,0.011048000305891037
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,127,0.01366879940032959
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,127,0.0150736004114151
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,127,0.013060800731182098
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,127,0.01305759996175766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,127,0.012883199751377106
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,127,0.01287200003862381
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,127,0.01326880007982254
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,127,0.013140800595283508
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,127,0.012857599556446076
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,127,0.012966400384902954
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,127,0.011076799780130386
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,127,0.011395200341939925
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,127,0.011059200018644333
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,127,0.01114400029182434
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,255,0.015031999349594117
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,255,0.013446399569511413
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,255,0.013806399703025819
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,255,0.013052800297737121
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,255,0.01180799975991249
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,255,0.013872000575065612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,255,0.012955200672149659
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,255,0.013316799700260163
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,255,0.012995199859142303
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,255,0.013494400680065155
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,255,0.011100800335407257
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,255,0.011075200140476226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,255,0.011086399853229522
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,255,0.011534400284290314
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,511,0.019499200582504272
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,511,0.017151999473571777
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,511,0.015447999536991119
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,511,0.015087999403476715
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,511,0.015131199359893798
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,511,0.015145599842071533
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,511,0.013155199587345123
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,511,0.015435199439525604
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,511,0.014617599546909332
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,511,0.013041600584983826
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,511,0.012833599746227265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,511,0.013206399977207184
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,511,0.013102400302886962
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,511,0.012854400277137756
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,1023,0.02529279887676239
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,1023,0.020803199708461763
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,1023,0.017679999768733978
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,1023,0.017059199512004852
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,1023,0.017004799842834473
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,1023,0.017366400361061095
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,1023,0.016859200596809388
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,1023,0.018939200043678283
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,1023,0.017046399414539337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,1023,0.015222400426864624
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,1023,0.014937600493431092
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,1023,0.01305759996175766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,1023,0.013044799864292144
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,2047,0.02196960002183914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,2047,0.027233600616455078
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,1023,0.013180799782276154
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,2047,0.021185599267482758
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,2047,0.01907680034637451
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,2047,0.01908479928970337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,2047,0.019246399402618408
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,2047,0.01910399943590164
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,2047,0.01918399930000305
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,2047,0.021187199652194975
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,2047,0.01740480065345764
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,2047,0.015091200172901154
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,2047,0.014983999729156493
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,2047,0.01494240015745163
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,2047,0.015006400644779205
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,4095,0.02784799933433533
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,4095,0.029446399211883544
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,4095,0.031167998909950256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,4095,0.021615999937057494
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,4095,0.021083199977874757
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,4095,0.021087999641895293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,4095,0.019270400702953338
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,4095,0.024059200286865236
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,4095,0.023233599960803986
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,4095,0.020848000049591066
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,4095,0.017051200568675994
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,4095,0.017433600127696992
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,4095,0.017347200214862822
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,4095,0.017003199458122252
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,8191,0.048588800430297854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,8191,0.05118880271911621
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,8191,0.054390400648117065
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,8191,0.0460640013217926
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,8191,0.045363199710845944
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,8191,0.04556959867477417
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,8191,0.04520640075206757
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,8191,0.03236800134181976
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,8191,0.027559998631477355
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,8191,0.030048000812530517
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,8191,0.02314079999923706
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,8191,0.02205760031938553
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,8191,0.021513600647449494
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,8191,0.02106720060110092
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,16383,0.07676640152931213
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,16383,0.07755360007286072
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,16383,0.09390239715576172
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,16383,0.07063199877738953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,16383,0.07092480063438415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,16383,0.07170079946517945
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,16383,0.07058719992637634
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,16383,0.05111839771270752
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,16383,0.04671199917793274
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,16383,0.05161280035972595
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,16383,0.040721601247787474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,16383,0.03982079923152924
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,16383,0.039643201231956485
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,16383,0.03935840129852295
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,32767,0.13323359489440917
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,32767,0.12936320304870605
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,32767,0.12263360023498535
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,32767,0.17097120285034179
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,32767,0.12143199443817139
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,32767,0.12089600563049316
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,32767,0.12135839462280273
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,32767,0.08312000036239624
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,32767,0.06961119771003724
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,32767,0.08589280247688294
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,32767,0.06327360272407531
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,32767,0.061919999122619626
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,32767,0.0616752028465271
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,32767,0.06055999994277954
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,65535,0.2486095905303955
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,65535,0.22929279804229735
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,65535,0.32460799217224123
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,65535,0.21848959922790528
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,65535,0.21875360012054443
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,65535,0.21786880493164062
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,65535,0.216646409034729
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,65535,0.14292800426483154
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,65535,0.11686400175094605
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,65535,0.15720479488372802
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,65535,0.10513919591903687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,65535,0.10368319749832153
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,65535,0.10426239967346192
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,65535,0.10413919687271118
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,16,1,1,131071,0.4652895927429199
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,16,1,2,131071,0.427400016784668
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,16,1,4,131071,0.6306767940521241
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,16,1,8,131071,0.40720481872558595
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,16,1,16,131071,0.40660958290100097
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,16,1,32,131071,0.408462381362915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,16,1,64,131071,0.40862240791320803
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,16,1,2,131071,0.21027679443359376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,16,1,1,131071,0.2611759901046753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,16,1,4,131071,0.3007999897003174
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,16,1,8,131071,0.18880640268325805
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,16,1,16,131071,0.1857200026512146
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,16,1,32,131071,0.18523199558258058
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,16,1,64,131071,0.18575680255889893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,1,0.01499360054731369
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,1,0.0147599995136261
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,1,0.012875199317932129
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,1,0.013070400059223174
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,1,0.012998400628566742
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,1,0.012763200700283051
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,1,0.012878400087356568
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,1,0.014833599328994751
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,1,0.014983999729156493
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,1,0.012833599746227265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,1,0.012891200184822083
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,1,0.012988799810409546
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,1,0.01289760023355484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,1,0.012772800028324127
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,3,0.014977599680423736
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,3,0.01494400054216385
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,3,0.014345599710941315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,3,0.012905600666999816
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,3,0.01276479959487915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,3,0.012590399384498597
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,3,0.012892800569534301
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,3,0.014843200147151948
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,3,0.01308639943599701
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,3,0.013334399461746216
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,3,0.012878400087356568
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,3,0.012761600315570831
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,3,0.01119839996099472
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,3,0.012774400413036346
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,7,0.014945599436759948
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,7,0.014873600006103516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,7,0.013048000633716583
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,7,0.012996800243854523
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,7,0.012830400466918945
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,7,0.012827199697494508
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,7,0.012892800569534301
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,7,0.014776000380516052
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,7,0.013065600395202636
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,7,0.01271360069513321
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,7,0.01281599998474121
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,7,0.011043199896812439
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,7,0.011054400354623795
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,7,0.01096159964799881
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,15,0.014953599870204925
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,15,0.01493919938802719
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,15,0.013235199451446533
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,15,0.012958399951457977
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,15,0.01289760023355484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,15,0.013031999766826629
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,15,0.013262400031089782
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,15,0.014884799718856812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,15,0.012908799946308136
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,15,0.013035200536251068
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,15,0.012880000472068786
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,15,0.010920000076293946
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,15,0.01101600006222725
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,15,0.012244799733161926
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,31,0.015027199685573579
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,31,0.015015999972820281
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,31,0.012928000092506409
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,31,0.01295360028743744
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,31,0.012936000525951386
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,31,0.012894399464130402
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,31,0.014859199523925781
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,31,0.012716799974441528
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,31,0.014507199823856353
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,31,0.012708799540996551
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,31,0.01276639997959137
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,31,0.010927999764680863
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,31,0.011337599903345107
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,31,0.01091040000319481
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,63,0.01499360054731369
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,63,0.014953599870204925
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,63,0.0139055997133255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,63,0.01297760009765625
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,63,0.012863999605178833
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,63,0.012796799838542938
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,63,0.012971200048923492
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,63,0.014795200526714325
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,63,0.012751999497413635
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,63,0.012758399546146392
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,63,0.012775999307632447
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,63,0.010785599797964096
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,63,0.01090560033917427
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,63,0.010856000334024429
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,127,0.014867199957370758
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,127,0.01496960073709488
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,127,0.013979199528694152
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,127,0.012964800000190735
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,127,0.013011200726032257
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,127,0.012894399464130402
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,127,0.012919999659061432
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,127,0.014824000000953675
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,127,0.01286720037460327
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,127,0.013630400598049163
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,127,0.012753599882125854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,127,0.01287200003862381
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,127,0.010918399691581726
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,127,0.010943999886512757
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,255,0.016073599457740784
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,255,0.01491519957780838
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,255,0.01290079951286316
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,255,0.012899200618267059
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,255,0.012859199941158295
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,255,0.012939199805259705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,255,0.014932799339294433
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,255,0.012689599394798278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,255,0.012865599989891053
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,255,0.012889599800109864
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,255,0.012804800271987915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,255,0.010999999940395355
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,255,0.01085439994931221
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,255,0.012555199861526489
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,511,0.021107199788093566
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,511,0.01902720034122467
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,511,0.017023999989032746
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,511,0.016574400663375854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,511,0.015056000649929046
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,511,0.014990399777889251
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,511,0.015014399588108063
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,511,0.01928640007972717
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,511,0.014959999918937683
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,511,0.012955200672149659
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,511,0.012876799702644348
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,511,0.012923200428485871
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,511,0.012937599420547485
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,511,0.01281919926404953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,1023,0.023299199342727662
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,1023,0.025496000051498414
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,1023,0.02122559994459152
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,1023,0.017023999989032746
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,1023,0.01693120002746582
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,1023,0.017023999989032746
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,1023,0.017051200568675994
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,1023,0.021052800118923187
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,1023,0.018985599279403687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,1023,0.017022399604320525
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,1023,0.014920000731945039
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,1023,0.015052799880504609
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,1023,0.014873600006103516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,1023,0.014868800342082978
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,2047,0.03149600028991699
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,2047,0.027300798892974855
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,2047,0.03134720027446747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,2047,0.021315200626850127
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,2047,0.021129600703716278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,2047,0.019278399646282196
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,2047,0.020763200521469117
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,2047,0.025276800990104674
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,2047,0.02102400064468384
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,2047,0.02096640020608902
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,2047,0.01679680049419403
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,2047,0.01695519983768463
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,2047,0.014934399724006652
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,2047,0.015049600601196289
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,4095,0.050440001487731936
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,4095,0.05021920204162598
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,4095,0.05556960105895996
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,4095,0.043844801187515256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,4095,0.04175199866294861
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,4095,0.042577600479125975
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,4095,0.04356479942798615
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,4095,0.0337007999420166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,4095,0.026020801067352294
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,4095,0.02942720055580139
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,4095,0.021137599647045136
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,4095,0.019382399320602418
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,4095,0.019047999382019044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,4095,0.020193600654602052
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,8191,0.07981119751930237
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,8191,0.0762000024318695
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,8191,0.09409120082855224
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,8191,0.06848959922790528
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,8191,0.0686847984790802
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,8191,0.0684719979763031
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,8191,0.06740639805793762
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,8191,0.05439199805259705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,8191,0.046342399716377256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,8191,0.05157279968261719
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,8191,0.03776159882545471
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,8191,0.03744319975376129
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,8191,0.037574398517608645
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,8191,0.03599199950695038
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,16383,0.13633760213851928
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,16383,0.13541280031204223
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,16383,0.12339680194854737
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,16383,0.17187360525131226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,16383,0.12245919704437255
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,16383,0.12323360443115235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,16383,0.12189120054244995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,16383,0.08393759727478027
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,16383,0.07220159769058228
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,16383,0.060502398014068606
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,16383,0.08729599714279175
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,16383,0.059324800968170166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,16383,0.059864002466201785
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,16383,0.05826079845428467
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,32767,0.24861760139465333
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,32767,0.2354111909866333
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,32767,0.32493760585784914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,32767,0.22321600914001466
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,32767,0.22076160907745362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,32767,0.22060799598693848
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,32767,0.2230976104736328
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,32767,0.1445248007774353
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,32767,0.12069599628448487
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,32767,0.1584272027015686
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,32767,0.10540319681167602
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,32767,0.10306240320205688
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,32767,0.10237280130386353
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,32767,0.10274080038070679
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,32,1,1,65535,0.47763681411743164
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,32,1,2,65535,0.43987360000610354
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,32,1,4,65535,0.6280975818634034
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,32,1,8,65535,0.42330079078674315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,32,1,16,65535,0.4223423957824707
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,32,1,32,65535,0.4220384120941162
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,32,1,64,65535,0.4224527835845947
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,32,1,1,65535,0.26187360286712646
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,32,1,2,65535,0.22138240337371826
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,32,1,4,65535,0.2958672046661377
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,32,1,8,65535,0.19394559860229493
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,32,1,16,65535,0.19055839776992797
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,32,1,32,65535,0.19197920560836793
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,32,1,64,65535,0.19062720537185668
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,1,0.014720000326633453
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,1,0.016998399794101716
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,1,0.015027199685573579
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,1,0.012964800000190735
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,1,0.012910400331020356
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,1,0.012918399274349212
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,1,0.012936000525951386
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,1,0.01688160002231598
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,1,0.014281600713729858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,1,0.014905600249767304
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,1,0.012936000525951386
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,1,0.012756800651550293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,1,0.01279039978981018
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,1,0.012942400574684144
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,3,0.016923199594020843
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,3,0.0147599995136261
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,3,0.014740799367427827
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,3,0.012931199371814727
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,3,0.012948800623416901
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,3,0.012734399735927581
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,3,0.016971200704574585
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,3,0.012759999930858612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,3,0.012806400656700134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,3,0.012915199995040894
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,3,0.0130048006772995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,3,0.012755200266838074
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,3,0.012598399817943574
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,3,0.01111840009689331
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,7,0.016867199540138246
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,7,0.014796799421310425
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,7,0.015059199929237366
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,7,0.012868799269199371
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,7,0.012943999469280243
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,7,0.012945599853992462
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,7,0.01295360028743744
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,7,0.016889600455760954
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,7,0.014793600142002105
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,7,0.0130048006772995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,7,0.012875199317932129
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,7,0.012678399682044983
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,7,0.012671999633312225
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,7,0.011787199974060058
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,15,0.016859200596809388
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,15,0.01483519971370697
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,15,0.014924800395965577
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,15,0.012998400628566742
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,15,0.012972800433635712
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,15,0.012945599853992462
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,15,0.012936000525951386
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,15,0.01688160002231598
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,15,0.012923200428485871
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,15,0.012963199615478515
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,15,0.012913599610328674
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,15,0.010847999900579452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,15,0.012740799784660339
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,15,0.0110944002866745
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,31,0.016843199729919434
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,31,0.014791999757289887
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,31,0.014951999485492706
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,31,0.013350400328636169
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,31,0.012934400141239167
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,31,0.013096000254154205
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,31,0.013105599582195282
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,31,0.016166399419307708
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,31,0.012889599800109864
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,31,0.014251199364662171
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,31,0.012723200023174286
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,31,0.010843200236558914
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,31,0.012745599448680877
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,31,0.012724800407886505
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,63,0.017001600563526155
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,63,0.014849600195884705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,63,0.01499519944190979
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,63,0.014961600303649902
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,63,0.012835200130939483
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,63,0.012884800136089326
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,63,0.013073599338531494
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,63,0.016734400391578676
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,63,0.01279360055923462
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,63,0.012873600423336028
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,63,0.012708799540996551
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,63,0.012671999633312225
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,63,0.012720000743865967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,63,0.010937599837779999
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,127,0.016631999611854555
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,127,0.014608000218868256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,127,0.015107199549674988
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,127,0.013928000628948212
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,127,0.012838399410247803
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,127,0.012960000336170197
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,127,0.013118399679660797
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,127,0.016812799870967864
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,127,0.012863999605178833
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,127,0.012825599312782288
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,127,0.012912000715732574
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,127,0.01281599998474121
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,127,0.012564800679683685
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,127,0.010811199992895126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,255,0.017632000148296356
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,255,0.01526239961385727
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,255,0.014856000244617463
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,255,0.012889599800109864
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,255,0.012943999469280243
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,255,0.012958399951457977
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,255,0.01680160015821457
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,255,0.012785600125789642
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,255,0.015113599598407745
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,255,0.013916799426078796
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,255,0.012041600048542022
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,255,0.012777599692344665
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,255,0.012675200402736665
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,255,0.012744000554084778
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,511,0.019288000464439393
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,511,0.02314720004796982
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,511,0.018980799615383147
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,511,0.016964800655841827
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,511,0.01523360013961792
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,511,0.01687999963760376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,511,0.016392000019550323
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,511,0.01896799951791763
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,511,0.020374399423599244
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,511,0.015017600357532501
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,511,0.013782399892807006
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,511,0.013025599718093871
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,511,0.01287200003862381
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,511,0.013078400492668152
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,1023,0.0274399995803833
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,1023,0.02932800054550171
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,1023,0.0293503999710083
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,1023,0.021076799929142
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,1023,0.018988800048828126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,1023,0.019070400297641753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,1023,0.01907680034637451
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,1023,0.023132799565792082
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,1023,0.02301120012998581
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,1023,0.019099199771881105
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,1023,0.017092800140380858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,1023,0.014732800424098969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,1023,0.014934399724006652
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,1023,0.015184000134468079
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,2047,0.0472463995218277
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,2047,0.05214880108833313
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,2047,0.053830397129058835
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,2047,0.044460800290107724
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,2047,0.04314239919185638
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,2047,0.043191999197006226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,2047,0.04388799965381622
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,2047,0.031118398904800414
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,2047,0.027371200919151305
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,2047,0.028655999898910524
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,2047,0.021297599375247955
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,2047,0.019172799587249757
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,2047,0.0190080001950264
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,2047,0.019118399918079378
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,4095,0.0760047972202301
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,4095,0.07927200198173523
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,4095,0.09232479929924012
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,4095,0.06791679859161377
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,4095,0.06917759776115417
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,4095,0.06829919815063476
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,4095,0.06851999759674073
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,4095,0.0487311989068985
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,4095,0.04799039959907532
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,4095,0.050379198789596555
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,4095,0.039505600929260254
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,4095,0.03747360110282898
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,4095,0.03747999966144562
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,4095,0.03670400083065033
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,8191,0.13159359693527223
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,8191,0.13691999912261962
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,8191,0.1701151967048645
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,8191,0.1236672043800354
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,8191,0.12277920246124267
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,8191,0.12291680574417115
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,8191,0.12299519777297974
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,8191,0.07892640233039856
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,8191,0.07393919825553893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,8191,0.08610720038414002
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,8191,0.062001597881317136
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,8191,0.05925920009613037
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,8191,0.05966240167617798
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,8191,0.059171199798583984
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,16383,0.24691998958587646
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,16383,0.23932480812072754
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,16383,0.3192991971969604
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,16383,0.2252608060836792
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,16383,0.22327520847320556
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,16383,0.2213599920272827
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,16383,0.22178239822387696
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,16383,0.13950560092926026
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,16383,0.12410399913787842
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,16383,0.15591200590133666
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,16383,0.10531840324401856
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,16383,0.10443999767303466
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,16383,0.10357439517974854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,16383,0.10270400047302246
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,64,1,1,32767,0.4643455982208252
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,64,1,2,32767,0.4449359893798828
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,64,1,4,32767,0.6200448036193847
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,64,1,8,32767,0.4230544090270996
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,64,1,16,32767,0.4224815845489502
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,64,1,32,32767,0.4231391906738281
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,64,1,64,32767,0.42276802062988283
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,64,1,2,32767,0.22197439670562744
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,64,1,1,32767,0.25639519691467283
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,64,1,4,32767,0.29617919921875
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,64,1,8,32767,0.19455519914627076
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,64,1,16,32767,0.1910320043563843
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,64,1,32,32767,0.19088159799575805
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,64,1,64,32767,0.1912384033203125
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,1,0.021027199923992157
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,1,0.01775680035352707
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,1,0.021118399500846863
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,1,0.014718399941921234
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,1,0.014894400537014008
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,1,0.014767999947071075
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,1,0.014443199336528777
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,1,0.023240000009536743
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,1,0.016939200460910797
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,1,0.017763200402259826
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,1,0.013017599284648896
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,1,0.012751999497413635
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,1,0.01281919926404953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,1,0.01271360069513321
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,3,0.02167679965496063
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,3,0.017057600617408752
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,3,0.014849600195884705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,3,0.02096640020608902
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,3,0.01488800048828125
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,3,0.014750400185585022
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,3,0.014937600493431092
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,3,0.023083199560642243
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,3,0.01684959977865219
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,3,0.018984000384807586
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,3,0.012940800189971924
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,3,0.012942400574684144
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,3,0.01276959925889969
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,3,0.012721599638462066
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,7,0.021486400067806243
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,7,0.017113600671291352
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,7,0.020985600352287293
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,7,0.014990399777889251
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,7,0.01483519971370697
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,7,0.01491519957780838
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,7,0.014884799718856812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,7,0.02306399941444397
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,7,0.016956800222396852
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,7,0.01913280040025711
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,7,0.012915199995040894
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,7,0.012870399653911591
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,7,0.012835200130939483
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,7,0.01313759982585907
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,15,0.021273599565029146
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,15,0.01698720008134842
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,15,0.021172800660133363
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,15,0.015059199929237366
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,15,0.014822399616241455
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,15,0.014920000731945039
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,15,0.014846399426460266
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,15,0.023217600584030152
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,15,0.01691199988126755
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,15,0.018915200233459474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,15,0.012971200048923492
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,15,0.012958399951457977
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,15,0.01273919939994812
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,15,0.012694400548934937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,31,0.021687999367713928
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,31,0.016982400417327882
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,31,0.020924800634384157
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,31,0.014899200201034546
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,31,0.014908799529075622
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,31,0.014776000380516052
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,31,0.014881600439548493
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,31,0.023080000281333925
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,31,0.01709440052509308
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,31,0.018878400325775146
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,31,0.012825599312782288
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,31,0.012689599394798278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,31,0.012809599936008453
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,31,0.012785600125789642
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,63,0.016952000558376312
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,63,0.022177599370479584
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,63,0.021132799983024596
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,63,0.015011200308799743
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,63,0.014766399562358857
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,63,0.0147599995136261
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,63,0.01518079936504364
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,63,0.023081600666046143
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,63,0.016972799599170686
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,63,0.018827199935913086
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,63,0.01494079977273941
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,63,0.012772800028324127
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,63,0.012780800461769104
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,63,0.012787200510501862
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,127,0.021748800575733186
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,127,0.017004799842834473
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,127,0.021035200357437132
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,127,0.014894400537014008
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,127,0.014788800477981567
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,127,0.014870400726795196
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,127,0.014839999377727509
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,127,0.022091199457645417
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,127,0.01687040030956268
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,127,0.012849600613117218
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,127,0.018884800374507904
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,127,0.012715199589729309
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,127,0.012838399410247803
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,127,0.012699200212955475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,255,0.027214398980140685
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,255,0.01900479942560196
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,255,0.02106879949569702
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,255,0.014723199605941772
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,255,0.01467519998550415
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,255,0.014822399616241455
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,255,0.015112000703811645
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,255,0.024988800287246704
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,255,0.017046399414539337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,255,0.018646399676799773
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,255,0.014057600498199463
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,255,0.012736000120639801
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,255,0.012641599774360657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,255,0.01281760036945343
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,511,0.03761279881000519
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,511,0.02633439898490906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,511,0.031656000018119815
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,511,0.019601599872112276
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,511,0.01894879937171936
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,511,0.018862399458885192
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,511,0.017056000232696534
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,511,0.029281601309776306
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,511,0.020895999670028687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,511,0.01480640023946762
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,511,0.014905600249767304
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,511,0.02104160040616989
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,511,0.01485760062932968
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,511,0.013572800159454345
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,1023,0.05384160280227661
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,1023,0.049502399563789365
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,1023,0.05606560111045837
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,1023,0.04392479956150055
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,1023,0.04169600009918213
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,1023,0.04149599969387054
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,1023,0.04158560037612915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,1023,0.039750400185585025
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,1023,0.026713600754737853
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,1023,0.030668801069259642
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,1023,0.019223999977111817
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,1023,0.0188960000872612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,1023,0.01903519928455353
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,1023,0.018606400489807128
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,2047,0.08532480001449586
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,2047,0.07522240281105042
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,2047,0.0954912006855011
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,2047,0.06787999868392944
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,2047,0.06837599873542785
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,2047,0.0655023992061615
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,2047,0.05623679757118225
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,2047,0.0685920000076294
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,2047,0.04730400145053863
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,2047,0.05334240198135376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,2047,0.03767040073871612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,2047,0.03598560094833374
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,2047,0.03690719902515412
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,2047,0.03572640120983124
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,4095,0.1404528021812439
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,4095,0.13522399663925172
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,4095,0.17267199754714965
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,4095,0.12141120433807373
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,4095,0.1201583981513977
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,4095,0.12119519710540771
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,4095,0.11849280595779418
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,4095,0.08872640132904053
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,4095,0.0718384027481079
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,4095,0.09065600037574768
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,4095,0.06017439961433411
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,4095,0.05826560258865356
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,4095,0.058166402578353885
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,4095,0.05857279896736145
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,8191,0.2505919933319092
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,8191,0.23566720485687256
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,8191,0.326475191116333
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,8191,0.22224159240722657
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,8191,0.22043840885162352
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,8191,0.21782240867614747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,8191,0.21822400093078614
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,8191,0.15212639570236205
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,8191,0.12028800249099732
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,8191,0.16066559553146362
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,8191,0.10469119548797608
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,8191,0.1038159966468811
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,8191,0.10173920392990113
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,8191,0.10252640247344971
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,128,1,1,16383,0.4760767936706543
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,128,1,2,16383,0.4399600028991699
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,128,1,4,16383,0.6274608135223388
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,128,1,8,16383,0.421233606338501
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,128,1,16,16383,0.4178624153137207
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,128,1,32,16383,0.41916799545288086
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,128,1,1,16383,0.269702410697937
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,128,1,64,16383,0.4170639991760254
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,128,1,2,16383,0.21871840953826904
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,128,1,4,16383,0.30231680870056155
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,128,1,8,16383,0.19147839546203613
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,128,1,16,16383,0.19078240394592286
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,128,1,32,16383,0.18911999464035034
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,128,1,64,16383,0.19022239446640016
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,1,0.03704000115394592
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,1,0.025172799825668335
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,1,0.03127520084381104
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,1,0.021054400503635405
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,1,0.019215999543666838
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,1,0.020729599893093108
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,1,0.019838400185108185
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,1,0.035576000809669495
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,1,0.023175999522209167
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,1,0.027208000421524048
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,1,0.01899999976158142
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,1,0.017015999555587767
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,1,0.01679359972476959
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,1,0.016924799978733064
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,3,0.03768480122089386
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,3,0.025089600682258607
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,3,0.03133600056171417
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,3,0.021244800090789794
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,3,0.020520000159740447
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,3,0.020715199410915375
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,3,0.020239999890327452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,3,0.035755199193954465
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,3,0.025113600492477416
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,3,0.027385601401329042
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,3,0.01897120028734207
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,3,0.01685280054807663
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,3,0.016864000260829924
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,3,0.017073599994182585
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,7,0.037487998604774475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,7,0.025009599328041077
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,7,0.03130879998207092
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,7,0.021161599457263945
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,7,0.019864000380039215
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,7,0.020751999318599702
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,7,0.020844799280166627
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,7,0.037038400769233704
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,7,0.023732799291610717
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,7,0.027137601375579835
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,7,0.019049599766731262
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,7,0.01721920073032379
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,7,0.01687999963760376
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,7,0.01717119961977005
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,15,0.03772799968719483
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,15,0.025303998589515687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,15,0.03132160007953644
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,15,0.021155199408531188
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,15,0.021011200547218323
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,15,0.021201600134372712
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,15,0.02035039961338043
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,15,0.03732799887657166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,15,0.02507199943065643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,15,0.01875839978456497
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,15,0.027369600534439088
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,15,0.016976000368595125
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,15,0.016864000260829924
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,15,0.016908800601959227
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,31,0.03947679996490479
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,31,0.025179201364517213
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,31,0.031564798951148984
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,31,0.02113119959831238
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,31,0.02104160040616989
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,31,0.021030400693416596
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,31,0.019177600741386414
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,31,0.037376001477241516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,31,0.02326720058917999
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,31,0.027321600914001466
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,31,0.018783999979496
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,31,0.0172447994351387
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,31,0.016923199594020843
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,31,0.016916799545288085
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,63,0.03871839940547943
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,63,0.02524479925632477
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,63,0.03155040144920349
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,63,0.02109919935464859
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,63,0.020403200387954713
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,63,0.021048000454902648
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,63,0.021139200031757354
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,63,0.03752320110797882
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,63,0.023137600719928743
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,63,0.02743520140647888
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,63,0.018985599279403687
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,63,0.017051200568675994
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,63,0.016892799735069276
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,63,0.01698720008134842
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,127,0.038446399569511416
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,127,0.025577598810195924
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,127,0.031455999612808226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,127,0.02109439969062805
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,127,0.021180799603462218
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,127,0.021135999262332915
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,127,0.020883199572563172
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,127,0.03750720024108887
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,127,0.023236800730228425
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,127,0.02733440101146698
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,127,0.018904000520706177
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,127,0.016913600265979767
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,127,0.01698720008134842
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,127,0.01695840060710907
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,255,0.04782080054283142
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,255,0.037662398815155027
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,255,0.03626399934291839
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,255,0.021860800683498383
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,255,0.020262399315834047
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,255,0.021060800552368163
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,255,0.020865599811077117
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,255,0.041740798950195314
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,255,0.027399998903274537
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,255,0.027480000257492067
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,255,0.0191551998257637
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,255,0.016953599452972413
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,255,0.01682559996843338
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,255,0.017115199565887453
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,511,0.06329280138015747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,511,0.05437279939651489
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,511,0.058671998977661136
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,511,0.04450559914112091
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,511,0.04159519970417023
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,511,0.04129279851913452
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,511,0.04168640077114105
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,511,0.05168799757957458
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,511,0.038790398836135866
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,511,0.03734720051288605
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,511,0.02294880002737045
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,511,0.02065120041370392
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,511,0.021001599729061127
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,511,0.01984640061855316
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,1023,0.09358400106430054
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,1023,0.08458560109138488
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,1023,0.0980351984500885
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,1023,0.06978080272674561
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,1023,0.06880159974098206
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,1023,0.06775519847869874
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,1023,0.06683359742164612
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,1023,0.06777600049972535
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,1023,0.0525488018989563
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,1023,0.0574832022190094
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,1023,0.03994239866733551
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,1023,0.0374096006155014
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,1023,0.03753440082073212
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,1023,0.03794400095939636
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,2047,0.15393120050430298
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,2047,0.13862240314483643
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,2047,0.1771504044532776
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,2047,0.1201200008392334
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,2047,0.12116800546646118
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,2047,0.11794240474700927
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,2047,0.11805119514465331
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,2047,0.10233119726181031
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,2047,0.08123520016670227
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,2047,0.06452159881591797
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,2047,0.0935696005821228
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,2047,0.06165120005607605
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,2047,0.06068639755249024
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,2047,0.06048160195350647
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,4095,0.2666768074035645
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,4095,0.2510240077972412
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,4095,0.33151841163635254
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,4095,0.22824320793151856
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,4095,0.2251375913619995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,4095,0.22484641075134276
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,4095,0.22360479831695557
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,4095,0.16473759412765504
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,4095,0.13277120590209962
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,4095,0.16897120475769042
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,4095,0.10984959602355956
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,4095,0.10640159845352173
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,4095,0.10606399774551392
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,4095,0.10592800378799438
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,256,1,1,8191,0.4848015785217285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,256,1,2,8191,0.46091198921203613
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,256,1,4,8191,0.6351119995117187
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,256,1,8,8191,0.43054399490356443
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,256,1,16,8191,0.4253392219543457
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,256,1,32,8191,0.42433757781982423
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,256,1,64,8191,0.4234767913818359
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,256,1,1,8191,0.2915119886398315
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,256,1,2,8191,0.23083040714263917
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,256,1,4,8191,0.3076575994491577
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,256,1,8,8191,0.19987679719924928
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,256,1,16,8191,0.1942304015159607
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,256,1,32,8191,0.19467200040817262
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,256,1,64,8191,0.19406720399856567
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,1,0.06496319770812989
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,1,0.043926399946212766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,1,0.03123680055141449
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,1,0.05165600180625916
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,1,0.02929919958114624
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,1,0.0295199990272522
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,1,0.029198399186134337
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,1,0.06462240219116211
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,1,0.03960799872875213
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,1,0.04376319944858551
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,1,0.02707200050354004
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,1,0.02500480115413666
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,1,0.0233024001121521
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,1,0.023503999412059783
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,3,0.06409919857978821
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,3,0.043747198581695554
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,3,0.05177119970321655
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,3,0.03321279883384705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,3,0.029364800453186034
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,3,0.029329600930213928
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,3,0.03041119873523712
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,3,0.06607040166854858
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,3,0.039505600929260254
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,3,0.04353759884834289
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,3,0.02723200023174286
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,3,0.025
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,3,0.023004800081253052
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,3,0.022758400440216063
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,7,0.0654416024684906
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,7,0.052193599939346316
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,7,0.03146719932556152
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,7,0.030964800715446474
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,7,0.04362399876117706
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,7,0.029374399781227113
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,7,0.02916319966316223
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,7,0.06616960167884826
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,7,0.03985599875450134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,7,0.04357919991016388
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,7,0.027167999744415285
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,7,0.02502560019493103
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,7,0.02510559856891632
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,7,0.023068800568580627
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,15,0.04510399997234345
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,15,0.06443840265274048
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,15,0.05262399911880493
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,15,0.032576000690460204
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,15,0.03126719892024994
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,15,0.029275199770927428
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,15,0.03128640055656433
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,15,0.06588799953460693
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,15,0.03936479985713959
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,15,0.043561598658561705
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,15,0.02728799879550934
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,15,0.025179201364517213
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,15,0.023083199560642243
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,15,0.02489600032567978
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,31,0.06479679942131042
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,31,0.04572480022907257
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,31,0.05186399817466736
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,31,0.033180800080299375
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,31,0.031179198622703554
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,31,0.03110080063343048
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,31,0.029342401027679443
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,31,0.06634719967842102
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,31,0.04038879871368408
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,31,0.0435696005821228
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,31,0.027300798892974855
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,31,0.02508159875869751
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,31,0.025051200389862062
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,31,0.023240000009536743
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,63,0.06544960141181946
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,63,0.04575999975204468
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,63,0.05410720109939575
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,63,0.03333599865436554
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,63,0.03097119927406311
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,63,0.029371199011802674
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,63,0.03134239912033081
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,63,0.06623200178146363
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,63,0.04147680103778839
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,63,0.043721601366996765
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,63,0.027241599559783936
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,63,0.025040000677108765
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,63,0.023102399706840516
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,63,0.023156799376010895
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,127,0.06798400282859803
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,127,0.04964320063591003
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,127,0.0562175989151001
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,127,0.03555200099945068
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,127,0.03130879998207092
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,127,0.02932479977607727
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,127,0.029707199335098265
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,127,0.06630719900131225
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,127,0.04325119853019714
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,127,0.043675199151039124
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,127,0.027267199754714967
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,127,0.02536959946155548
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,127,0.024937599897384644
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,127,0.023188799619674683
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,255,0.08436639904975891
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,255,0.060899198055267334
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,255,0.0656063973903656
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,255,0.04780640006065369
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,255,0.04352160096168518
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,255,0.043296000361442565
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,255,0.04245760142803192
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,255,0.07032319903373718
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,255,0.050723201036453246
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,255,0.047035199403762815
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,255,0.02928000092506409
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,255,0.02500160038471222
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,255,0.023873600363731384
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,255,0.023656000196933747
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,511,0.11042879819869995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,511,0.095278400182724
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,511,0.09732800126075744
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,511,0.0773360013961792
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,511,0.07308319807052613
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,511,0.07126719951629638
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,511,0.07053279876708984
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,511,0.08697919845581055
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,511,0.06594079732894897
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,511,0.061559998989105226
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,511,0.045407998561859134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,511,0.04146080017089844
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,511,0.03975679874420166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,511,0.03948639929294586
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,1023,0.16344640254974366
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,1023,0.15021920204162598
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,1023,0.16819039583206177
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,1023,0.1260800004005432
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,1023,0.12218400239944457
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,1023,0.11976319551467896
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,1023,0.1200943946838379
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,1023,0.11421279907226563
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,1023,0.09256160259246826
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,1023,0.09587519764900207
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,1023,0.06881120204925537
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,1023,0.06420000195503235
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,1023,0.06267679929733276
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,1023,0.06267359852790833
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,2047,0.272542405128479
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,2047,0.2574095964431763
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,2047,0.30601599216461184
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,2047,0.22866721153259278
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,2047,0.2195904016494751
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,2047,0.22068159580230712
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,2047,0.22001440525054933
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,2047,0.17560479640960694
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,2047,0.1479024052619934
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,2047,0.1610592007637024
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,2047,0.1147312045097351
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,2047,0.11148960590362549
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,2047,0.10913759469985962
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,2047,0.10767199993133544
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,512,1,1,4095,0.474945592880249
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,512,1,2,4095,0.4905712127685547
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,512,1,4,4095,0.5770639896392822
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,512,1,8,4095,0.44136481285095214
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,512,1,16,4095,0.43514561653137207
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,512,1,32,4095,0.43524961471557616
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,512,1,64,4095,0.4325839996337891
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,512,1,1,4095,0.2861135959625244
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,512,1,2,4095,0.2506160020828247
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,512,1,4,4095,0.29196479320526125
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,512,1,8,4095,0.20752320289611817
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,512,1,16,4095,0.20000479221343995
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,512,1,32,4095,0.19864959716796876
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,512,1,64,4095,0.19873119592666627
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,1,0.11767040491104126
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,1,0.07609440088272094
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,1,0.09710239768028259
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,1,0.0528656005859375
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,1,0.049620801210403444
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,1,0.047681599855422974
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,1,0.04782879948616028
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,1,0.11984800100326538
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,1,0.07066720128059387
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,1,0.07829920053482056
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,1,0.04561919867992401
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,1,0.03837920129299164
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,1,0.037401598691940305
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,1,0.036452800035476685
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,3,0.11864960193634033
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,3,0.07647519707679748
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,3,0.09688000082969665
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,3,0.053830397129058835
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,3,0.04781599938869476
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,3,0.04779039919376373
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,3,0.04782400131225586
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,3,0.11813759803771973
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,3,0.07067520022392274
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,3,0.07836639881134033
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,3,0.04553439915180206
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,3,0.03907040059566498
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,3,0.03740000128746033
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,3,0.037222400307655334
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,7,0.11863199472427369
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,7,0.07647200226783753
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,7,0.0970080018043518
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,7,0.05343679785728454
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,7,0.04956960082054138
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,7,0.047881600260734555
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,7,0.04784800112247467
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,7,0.1187391996383667
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,7,0.07067040205001832
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,7,0.07839679718017578
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,7,0.04547359943389893
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,7,0.037487998604774475
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,7,0.03741919994354248
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,7,0.0367935985326767
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,15,0.11924959421157837
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,15,0.07625600099563598
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,15,0.05417760014533997
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,15,0.04971039891242981
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,15,0.0969103991985321
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,15,0.04778240025043488
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,15,0.04864960014820099
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,15,0.11994240283966065
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,15,0.07152479887008667
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,15,0.07840319871902465
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,15,0.045668798685073855
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,15,0.03945919871330261
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,15,0.037564799189567566
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,15,0.03732799887657166
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,31,0.11783360242843628
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,31,0.07684479951858521
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,31,0.0973695993423462
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,31,0.05523040294647217
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,31,0.04975999891757965
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,31,0.04782719910144806
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,31,0.04819360077381134
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,31,0.1181872010231018
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,31,0.07131999731063843
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,31,0.07891520261764526
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,31,0.0456063985824585
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,31,0.037561601400375365
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,31,0.0375247985124588
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,31,0.03734399974346161
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,63,0.11983200311660766
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,63,0.07791200280189514
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,63,0.0971664011478424
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,63,0.05764960050582886
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,63,0.05164480209350586
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,63,0.04966239929199219
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,63,0.04855200052261353
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,63,0.11987520456314087
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,63,0.07281919717788696
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,63,0.08066239953041077
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,63,0.045526400208473206
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,63,0.03920960128307342
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,63,0.03745599985122681
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,63,0.03726080060005188
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,127,0.12172640562057495
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,127,0.08684160113334656
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,127,0.09961439967155457
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,127,0.06430240273475647
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,127,0.057124799489974974
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,127,0.05620480179786682
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,127,0.05592479705810547
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,127,0.11955360174179078
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,127,0.07234240174293519
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,127,0.08060640096664429
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,127,0.04965600073337555
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,127,0.040249601006507874
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,127,0.037441599369049075
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,127,0.03745599985122681
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,255,0.1504528045654297
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,255,0.11333600282669068
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,255,0.10723199844360351
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,255,0.07850080132484435
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,255,0.0716863989830017
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,255,0.06846399903297425
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,255,0.06807519793510437
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,255,0.12895679473876953
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,255,0.08592159748077392
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,255,0.07891839742660522
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,255,0.051881599426269534
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,255,0.044651201367378233
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,255,0.042844799160957334
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,255,0.042473599314689636
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,511,0.2014319896697998
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,511,0.16430879831314088
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,511,0.1781056046485901
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,511,0.13049919605255128
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,511,0.1232367992401123
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,511,0.1190335988998413
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,511,0.11955519914627075
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,511,0.1624719977378845
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,511,0.10935519933700562
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,511,0.11019999980926513
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,511,0.07282559871673584
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,511,0.06679360270500183
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,511,0.06425920128822327
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,511,0.06288480162620544
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,1023,0.307590389251709
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,1023,0.26685280799865724
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,1023,0.32042560577392576
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,1023,0.2206943988800049
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,1023,0.2131648063659668
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,1023,0.2109328031539917
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,1023,0.20739519596099854
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,1023,0.2208319902420044
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,1023,0.15922720432281495
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,1023,0.1759503960609436
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,1023,0.11570240259170532
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,1023,0.10930559635162354
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,1023,0.10487680435180664
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,1023,0.10542399883270263
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,128,1024,1,1,2047,0.5227952003479004
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,64,1024,1,2,2047,0.46239361763000486
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,32,1024,1,4,2047,0.5957359790802002
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,16,1024,1,8,2047,0.4115664005279541
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,8,1024,1,16,2047,0.3990880012512207
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,4,1024,1,32,2047,0.39536640644073484
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,float16,2,1024,1,64,2047,0.39260001182556153
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,128,1024,1,1,2047,0.337662410736084
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,64,1024,1,2,2047,0.25971360206604005
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,32,1024,1,4,2047,0.30585598945617676
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,16,1024,1,8,2047,0.2024463891983032
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,8,1024,1,16,2047,0.1953104019165039
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,4,1024,1,32,2047,0.19029279947280883
SGLang,0.5.9,NVIDIA GB200,mla_generation,trtllm_mla,float16,fp8,2,1024,1,64,2047,0.18860479593276977
