framework,version,device,op_name,kernel_source,bmm_dtype,num_tokens,num_heads,latency
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,128,0.012015999853610992
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,64,0.008102399855852127
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,32,0.006777600198984146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,16,0.006828799843788147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,8,0.006787200272083282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,4,0.006796800345182419
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,128,0.014326399564743042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,64,0.010556799918413162
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,32,0.006943999975919724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,16,0.006864000111818314
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,8,0.006924799829721451
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,4,0.006812799721956253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,2,0.007587199658155441
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,1,0.007123199850320816
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,128,0.014483200013637542
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,64,0.0106175996363163
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,32,0.007036799937486649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,16,0.006905599683523178
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,8,0.007187200337648391
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,4,0.006777600198984146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,2,0.0069472000002861025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,2,0.007196799665689468
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,1,0.0067552000284194945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,1,0.007036799937486649
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,128,0.014553600549697876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,64,0.010572800040245056
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,16,0.006803199648857117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,8,0.00769599974155426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,4,0.0067840002477169035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,2,0.007673600316047668
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,1,0.0067552000284194945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,128,0.014870400726795196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,64,0.010700800269842149
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,32,0.007353600114583969
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,16,0.006697600334882736
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,2,0.006700800359249115
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,1,0.006771200150251388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,128,0.01600960046052933
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,32,0.007699199765920639
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,32,0.0072512000799179075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,16,0.006707199662923813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,8,0.0067552000284194945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,4,0.006646399945020675
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,2,0.006777600198984146
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,1,0.0067840002477169035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,128,0.01615999937057495
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,64,0.011196800321340562
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,32,0.010316800326108932
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,16,0.007366400212049484
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,8,0.006825599819421768
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,4,0.007152000069618225
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,4,0.006844799965620041
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,2,0.006761600077152252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,1,0.006844799965620041
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,128,0.017628799378871917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,64,0.011462400108575821
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,32,0.010585600137710571
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,64,0.01103999987244606
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,8,0.006534399837255478
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,4,0.006732799857854843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,2,0.00676800012588501
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,1,0.006732799857854843
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,128,0.02297919988632202
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,64,0.01616320013999939
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,16,0.009139200299978256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,8,0.006819199770689011
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,4,0.006355199962854385
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,2,0.006787200272083282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,1,0.00650240033864975
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,128,0.025363200902938844
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,64,0.016451199352741242
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,32,0.013337600231170654
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,8,0.006547199934720993
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,16,0.009424000233411788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,8,0.006761600077152252
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,4,0.006374400109052658
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,2,0.006483200192451477
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,1,0.008291199803352356
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,128,0.030588799715042116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,64,0.017369599640369417
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,32,0.01629119962453842
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,8,0.007532799988985062
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,4,0.006508799642324448
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,2,0.0067552000284194945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,1,0.0064640000462532045
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,128,0.050937598943710326
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,64,0.023094399273395537
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,32,0.015827199816703795
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,16,0.011750400066375732
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,16,0.007507199794054032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,4,0.006665600091218948
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,2,0.006553599983453751
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,1,0.006499200314283371
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,128,0.05377920269966126
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,32,0.01324159950017929
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,64,0.023449599742889404
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,32,0.015894399583339693
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,16,0.011753600090742111
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,8,0.00896959975361824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,4,0.0069023996591567995
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,2,0.00671359971165657
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,1,0.006611199676990509
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,128,0.058329600095748904
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,64,0.024348799884319306
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,32,0.016297599673271178
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,16,0.011875200271606445
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,8,0.010620799660682679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,4,0.00748480036854744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,2,0.006729599833488464
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,16,0.010579200088977813
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,1,0.006630399823188781
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,64,0.03410559892654419
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,128,0.08881279826164246
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,32,0.019011199474334717
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,16,0.012198399752378464
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,8,0.011964800208806992
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,4,0.00820479989051819
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,2,0.006787200272083282
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,1,0.006457599997520447
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,128,0.0924448013305664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,64,0.0357151985168457
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,32,0.019257600605487823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,16,0.01213119998574257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,8,0.013526399433612824
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,4,0.009043200314044953
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,2,0.0068960003554821014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,1,0.006441599875688553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,64,0.04734080135822296
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,32,0.023980799317359924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,8,0.008115199953317642
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,8,0.01626559942960739
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,2,0.007561600208282471
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,4,0.010611200332641601
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,1,0.007971200346946716
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,128,0.1480831980705261
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,64,0.06771519780158997
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,32,0.03514240086078644
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,16,0.01918720006942749
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,8,0.013782399892807006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,4,0.008790399879217148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,1,0.006409599632024765
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,128,0.19197440147399902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,64,0.08614720106124878
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,32,0.041196799278259276
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,16,0.023948800563812257
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,8,0.017241600155830383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,4,0.011715199798345566
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,2,0.008300799876451492
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,1,0.00631679967045784
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,128,0.2837088108062744
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,64,0.12334400415420532
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,32,0.0649728000164032
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,16,0.0334879994392395
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,8,0.022623999416828154
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,4,0.013680000603199006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,2,0.008646400272846222
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,1,0.006886400282382965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,128,0.10236480236053466
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,16,0.016204799711704253
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,32,0.08296639919281006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,16,0.04144960045814514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,4,0.017561599612236023
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,8,0.028067201375961304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,2,0.011686400324106217
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,1,0.0080704003572464
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,128,0.5557695865631104
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,64,0.23585920333862304
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,16,0.06319360136985779
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,8,0.040336000919342044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,2,0.013820800185203552
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,4,0.022963200509548188
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,1,0.008767999708652496
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,64,0.30982398986816406
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,128,0.7337471961975097
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,32,0.15699520111083984
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,16,0.08286719918251037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,8,0.05207359790802002
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,4,0.0282368004322052
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,2,0.017603200674057008
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,1,0.011484800279140473
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,64,0.15897279977798462
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,64,0.4588031768798828
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,32,0.23286080360412598
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,128,0.9168767929077148
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,16,0.12129600048065185
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,8,0.06393280029296874
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,4,0.03364799916744232
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,2,0.019424000382423402
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,2,0.0068063996732234955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,128,0.374617600440979
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,32,0.30602240562438965
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,64,0.6110208034515381
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,16,0.15639040470123292
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,8,0.08209279775619507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,2,0.024249599874019624
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,4,0.04176320135593414
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,128,1.2186367988586426
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,1,0.01993599981069565
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,1,0.011788800358772278
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,32,0.12087039947509766
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,128,0.011635199934244157
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,64,0.008928000181913375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,32,0.0072512000799179075
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,16,0.006319999694824219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,8,0.0064351998269557955
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,4,0.006319999694824219
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,2,0.006281600147485733
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,1,0.006128000095486641
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,128,0.012697599828243256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,64,0.009647999703884125
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,32,0.007542400062084198
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,16,0.007004799693822861
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,8,0.006985600292682648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,4,0.006960000097751618
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,2,0.007039999961853028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,1,0.006828799843788147
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,128,0.012703999876976013
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,64,0.0095551997423172
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,32,0.007158400118350982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,16,0.007049600034952164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,8,0.0069920003414154054
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,4,0.006943999975919724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,2,0.0069311998784542085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,1,0.0069311998784542085
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,64,0.009539200365543366
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,32,0.0071552000939846035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,16,0.007174400240182876
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,8,0.006969600170850754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,4,0.0069760002195835115
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,2,0.006995200365781784
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,1,0.006764800101518631
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,128,0.012800000607967377
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,64,0.009487999975681305
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,32,0.007097599655389785
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,4,0.007049600034952164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,2,0.007030399888753891
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,1,0.006969600170850754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,128,0.012928000092506409
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,64,0.009875199943780898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,32,0.007158400118350982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,16,0.007039999961853028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,8,0.007049600034952164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,4,0.006943999975919724
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,2,0.006988800317049027
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,128,0.012880000472068786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,1,0.006921599805355072
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,128,0.013167999684810638
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,64,0.010259199887514114
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,32,0.007465600222349167
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,16,0.007091200351715088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,8,0.0071103997528553006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,4,0.007148800045251846
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,16,0.008380799740552902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,8,0.007273600250482559
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,1,0.006924799829721451
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,2,0.007260800153017044
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,32,0.007737600058317184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,16,0.0073472000658512115
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,8,0.007238399982452392
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,4,0.0072672002017498015
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,2,0.007107199728488922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,1,0.007107199728488922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,128,0.013715200126171112
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,128,0.021007999777793884
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,64,0.012857599556446076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,32,0.012716799974441528
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,16,0.007737600058317184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,8,0.00729919970035553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,4,0.007142399996519088
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,2,0.007276800274848938
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,1,0.0071263998746871945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,128,0.02358720004558563
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,64,0.014310400187969207
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,16,0.007868800312280655
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,8,0.007289600372314453
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,4,0.00713919997215271
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,2,0.0071263998746871945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,1,0.007158400118350982
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,64,0.014752000570297241
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,32,0.008851200342178345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,16,0.007852800190448761
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,8,0.007145600020885467
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,64,0.010828799754381179
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,4,0.007177600264549255
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,2,0.007107199728488922
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,1,0.007020799815654755
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,128,0.03775359988212586
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,64,0.021167999505996703
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,32,0.012812800705432892
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,16,0.008601599931716919
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,8,0.0074432000517845156
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,4,0.007264000177383423
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,1,0.007248000055551529
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,128,0.04010879993438721
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,64,0.02136320024728775
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,32,0.01292479932308197
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,16,0.008591999858617782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,8,0.007235199958086014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,4,0.007302399724721909
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,2,0.007103999704122543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,1,0.0071552000939846035
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,128,0.046876800060272214
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,64,0.021750399470329286
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,32,0.01313599944114685
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,16,0.01040000021457672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,8,0.008799999952316284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,4,0.00865280032157898
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,2,0.008662399649620057
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,1,0.007836800068616867
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,128,0.053635197877883914
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,64,0.027612799406051637
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,32,0.013471999764442444
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,32,0.008963199704885483
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,16,0.009478399902582169
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,128,0.03362880051136017
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,8,0.008822400122880936
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,2,0.008742400258779526
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,4,0.008713600039482117
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,1,0.007049600034952164
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,128,0.0586624026298523
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,64,0.03384000062942505
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,32,0.01494400054216385
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,16,0.010092800110578537
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,4,0.008879999816417693
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,8,0.00873280018568039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,2,0.008687999844551087
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,1,0.007081600278615952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,64,0.04284160137176514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,128,0.07254719734191895
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,32,0.021558399498462676
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,16,0.014771200716495514
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,8,0.008895999938249587
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,4,0.008752000331878663
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,1,0.007113599777221679
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,128,0.10364160537719727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,2,0.007283200323581695
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,32,0.02654080092906952
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,16,0.015360000729560851
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,8,0.009907200187444686
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,4,0.008806400001049042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,2,0.008748800307512284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,1,0.007334399968385696
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,64,0.06784319877624512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,32,0.039955198764801025
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,16,0.019766399264335634
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,8,0.014614400267601014
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,4,0.008931200206279754
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,2,0.008806400001049042
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,1,0.00729919970035553
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,128,0.1844640016555786
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,64,0.09424960017204284
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,32,0.05175359845161438
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,16,0.02680320143699646
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,8,0.014672000706195832
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,4,0.010291200131177902
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,2,0.008819200098514557
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,1,0.007388799637556076
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,128,0.2381983995437622
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,64,0.12279679775238037
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,32,0.06411839723587036
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,16,0.03947519958019256
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,8,0.021411199867725373
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,4,0.013043199479579926
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,2,0.008928000181913375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,1,0.007737600058317184
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,64,0.17533119916915893
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,128,0.35129599571228026
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,32,0.09220479726791382
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,16,0.051507198810577394
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,8,0.02215680032968521
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,2,0.00873280018568039
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,4,0.013420799374580383
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,2,0.010147199779748917
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,64,0.05384320020675659
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,64,0.23127999305725097
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,32,0.11966400146484375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,128,0.4657695770263672
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,16,0.06543359756469727
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,8,0.03926720023155213
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,128,0.13011519908905028
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,4,0.02125760018825531
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,2,0.014931200444698334
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,1,0.008937600255012512
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,64,0.3330015897750854
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,16,0.09347839951515198
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,128,0.6870975971221924
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,8,0.05077760219573975
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,4,0.02202879935503006
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,2,0.015849600732326507
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,1,0.010428799688816071
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,16,0.12200319766998291
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,32,0.23284800052642823
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,64,0.45159039497375486
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,8,0.06422079801559448
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,4,0.039273598790168764
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,2,0.021433599293231964
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,128,0.9088895797729493
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,64,0.6679552078247071
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,32,0.34634239673614503
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,16,0.17602880001068116
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,128,1.3682047843933105
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,1,0.008591999858617782
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,8,0.08897600173950196
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,4,0.05057920217514038
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,2,0.02240000069141388
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,1,0.013750399649143218
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,32,0.4524928092956543
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,32,0.17209600210189818
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,64,0.89486083984375
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,8,0.11687359809875489
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,4,0.06469119787216186
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,2,0.03890880048274994
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,1,0.019791999459266664
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,128,1.8414207458496095
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,1,0.012982399761676788
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,16,0.29369919300079345
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,8,0.14425599575042725
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,32,0.5612383842468261
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,64,1.1350175857543945
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,4,0.07644799947738648
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,2,0.04291839897632599
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,1,0.02054080069065094
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,128,2.243510437011719
SGLang,0.5.9,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,16,0.23676159381866455
