framework,version,device,op_name,kernel_source,bmm_dtype,num_tokens,num_heads,latency
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,128,0.012070400267839431
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,64,0.008131200075149536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,32,0.0069760002195835115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,16,0.006911999732255936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,8,0.007014399766921997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,4,0.006831999868154526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,2,0.006780800223350525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,1,0.006911999732255936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,128,0.014211200177669525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,64,0.010425599664449692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,32,0.007030399888753891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,16,0.00679360032081604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,8,0.00671359971165657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,4,0.006825599819421768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,2,0.006672000139951706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,1,0.006729599833488464
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,128,0.014396800100803376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,64,0.010499200224876404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,32,0.007116799801588058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,16,0.006748799979686737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,8,0.006569600105285645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,4,0.006703999638557434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,2,0.006870400160551071
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,1,0.006787200272083282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,64,0.010681600123643876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,128,0.014608000218868256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,32,0.007619199901819229
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,16,0.006758400052785873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,8,0.006982400268316269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,4,0.007065600156784058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,1,0.006854400038719177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,2,0.006745599955320358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,64,0.01080000028014183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,128,0.014870400726795196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,16,0.006854400038719177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,32,0.007507199794054032
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,8,0.029721599817276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,4,0.006588800251483918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,2,0.006540799885988236
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,1,0.006649599969387054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,128,0.015804800391197204
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,64,0.01093439981341362
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,32,0.007625599950551986
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,16,0.006630399823188781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,8,0.006601600348949433
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,4,0.006822399795055389
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,2,0.0071712002158164975
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,1,0.006838399916887283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,128,0.01642560064792633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,64,0.011283200234174728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,32,0.01043199971318245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,16,0.007545600086450577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,8,0.006719999760389328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,2,0.006595200300216675
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,1,0.007145600020885467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,128,0.017507199943065644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,64,0.011631999909877778
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,32,0.010764800012111664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,16,0.007680000364780426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,8,0.006665600091218948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,4,0.00655680000782013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,1,0.006623999774456024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,128,0.0234592005610466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,64,0.016492800414562227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,32,0.013331200182437896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,16,0.0091839998960495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,8,0.0068351998925209045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,4,0.0067840002477169035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,4,0.00658240020275116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,2,0.0066880002617835995
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,1,0.006611199676990509
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,128,0.02462400048971176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,64,0.016329599916934966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,32,0.013478399813175201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,16,0.009302400052547455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,8,0.00692799985408783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,4,0.006617599725723266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,2,0.006630399823188781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,2,0.006364800035953522
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,1,0.006348799914121628
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,128,0.030355200171470642
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,64,0.016982400417327882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,32,0.016332800686359405
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,16,0.01048320010304451
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,8,0.009462399780750275
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,4,0.008252800256013871
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,2,0.008329600095748901
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,1,0.008630400151014328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,128,0.05103359818458557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,64,0.023340800404548646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,32,0.016198399662971496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,16,0.012060800194740295
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,8,0.009385599941015243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,4,0.00958079993724823
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,2,0.006656000018119812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,1,0.006569600105285645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,128,0.05327039957046509
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,64,0.023443199694156647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,32,0.015958400070667268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,16,0.011699199676513672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,8,0.009027200192213059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,4,0.006703999638557434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,2,0.00663359984755516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,1,0.0066880002617835995
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,128,0.05786240100860596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,64,0.02411839962005615
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,32,0.016339200735092162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,16,0.011868800222873687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,8,0.010576000064611435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,4,0.007542400062084198
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,1,0.006623999774456024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,128,0.08881279826164246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,64,0.033107200264930726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,32,0.019014400243759156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,16,0.012223999947309494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,8,0.012064000219106674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,4,0.008188799768686295
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,2,0.0065311998128890995
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,1,0.0064640000462532045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,128,0.09260159730911255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,64,0.03718400001525879
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,32,0.019171200692653656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,16,0.01218239963054657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,8,0.013446399569511413
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,4,0.009174399822950364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,1,0.006499200314283371
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,128,0.10258879661560058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,64,0.046998399496078494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,32,0.023769600689411162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,16,0.016121600568294526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,8,0.01611520051956177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,4,0.01037440001964569
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,2,0.007574400305747986
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,1,0.006377600133419037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,128,0.1495903968811035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,2,0.0069472000002861025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,64,0.06759039759635925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,32,0.033267199993133545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,16,0.019120000302791595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,8,0.013724799454212188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,4,0.008511999994516373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,2,0.006985600292682648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,64,0.08656319975852966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,128,0.19071359634399415
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,32,0.04216960072517395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,16,0.023897600173950196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,4,0.01167680025100708
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,8,0.01721920073032379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,2,0.008275199681520462
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,1,0.006521599739789963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,128,0.2844736099243164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,64,0.12472000122070312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,32,0.0648639976978302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,16,0.033769598603248595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,2,0.006892800331115723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,8,0.022710399329662324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,2,0.008636800199747085
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,4,0.013699199259281158
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,1,0.006796800345182419
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,64,0.15981760025024414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,128,0.3735775947570801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,32,0.08283519744873047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,16,0.042531201243400575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,8,0.02800639867782593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,4,0.01722559928894043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,2,0.011692799627780914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,1,0.007952000200748443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,64,0.23526721000671386
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,32,0.12066559791564942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,16,0.06303359866142273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,128,0.5543712139129638
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,8,0.04011200070381164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,2,0.01353600025177002
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,4,0.022579200565814972
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,1,0.008582399785518646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,64,0.3100064039230347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,32,0.1565600037574768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,16,0.082505601644516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,8,0.05183359980583191
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,4,0.027833598852157592
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,128,0.7320384025573731
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,1,0.011260800063610077
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,2,0.017472000420093538
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,64,0.45859518051147463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,16,0.11989120244979859
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,8,0.06384320259094238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,32,0.23212480545043945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,4,0.03339200019836426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,1,0.0067103996872901915
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,2,0.019280000030994414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,1,0.011638399958610535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,128,0.9160160064697266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,16,0.15560959577560424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,32,0.305839991569519
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,64,0.6085343837738038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,4,0.04113920032978058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,2,0.024105599522590636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,1,0.019817599654197694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,128,1.2159263610839843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,8,0.0827072024345398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,128,0.011641599982976914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,64,0.008799999952316284
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,32,0.00740479975938797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,16,0.006723199784755707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,8,0.006681600213050842
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,4,0.006672000139951706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,2,0.007030399888753891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,1,0.0068063996732234955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,128,0.012671999633312225
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,64,0.00952640026807785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,32,0.007193599641323089
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,16,0.007142399996519088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,8,0.007097599655389785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,4,0.0069983996450901035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,2,0.007599999755620956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,1,0.0067840002477169035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,128,0.01268800050020218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,64,0.009603200107812881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,32,0.007199999690055847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,16,0.00695360004901886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,8,0.006934399902820587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,4,0.007056000083684922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,2,0.0068000003695487974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,1,0.0068800002336502075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,128,0.012665599584579468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,64,0.009542399644851684
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,32,0.007113599777221679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,16,0.007091200351715088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,8,0.006915199756622315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,4,0.006867200136184692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,1,0.006815999746322632
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,128,0.01286720037460327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,64,0.009455999732017517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,32,0.0072512000799179075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,8,0.007030399888753891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,4,0.0069920003414154054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,2,0.0069920003414154054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,1,0.00700799971818924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,128,0.012950399518013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,64,0.01016639992594719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,32,0.00721919983625412
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,16,0.0071552000939846035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,4,0.007004799693822861
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,2,0.006908799707889557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,1,0.006867200136184692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,64,0.01019200012087822
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,32,0.007532799988985062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,16,0.0071552000939846035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,8,0.007152000069618225
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,4,0.007833600044250488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,2,0.007052800059318543
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,1,0.006860800087451935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,128,0.013478399813175201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,64,0.010841599851846694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,32,0.007718399912118912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,16,0.007235199958086014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,8,0.007184000313282013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,4,0.007062400132417679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,2,0.007132799923419952
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,2,0.007020799815654755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,1,0.006995200365781784
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,128,0.02088959962129593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,64,0.012787200510501862
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,32,0.012614400684833526
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,16,0.007561600208282471
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,128,0.01324159950017929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,8,0.007011199742555619
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,16,0.007152000069618225
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,8,0.00713919997215271
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,4,0.007030399888753891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,2,0.007068800181150437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,1,0.0069920003414154054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,128,0.024985599517822265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,64,0.014368000626564025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,32,0.0088639996945858
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,16,0.007577600330114365
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,8,0.007184000313282013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,4,0.007145600020885467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,2,0.007107199728488922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,1,0.007068800181150437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,128,0.033609598875045776
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,64,0.014441600441932679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,32,0.008819200098514557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,16,0.007670400291681289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,8,0.00719040036201477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,4,0.007065600156784058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,2,0.007091200351715088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,1,0.007081600278615952
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,128,0.03742400109767914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,64,0.02102400064468384
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,32,0.012809599936008453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,16,0.008444800227880477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,8,0.007248000055551529
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,4,0.007100799679756164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,2,0.0070880003273487095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,1,0.007199999690055847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,128,0.040566399693489075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,64,0.021087999641895293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,32,0.01284479945898056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,16,0.008716800063848496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,8,0.007344000041484833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,4,0.007254400104284286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,2,0.00719040036201477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,1,0.0072223998606204985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,128,0.04586879909038544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,64,0.021513600647449494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,32,0.012860800325870513
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,16,0.008902399986982345
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,8,0.008691199868917466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,4,0.008617600053548813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,2,0.008703999966382981
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,1,0.006995200365781784
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,128,0.05262399911880493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,64,0.029491201043128967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,32,0.013299199938774108
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,16,0.009859199821949004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,8,0.008803199976682663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,4,0.008720000088214875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,2,0.00859839990735054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,1,0.007052800059318543
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,128,0.058192002773284915
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,64,0.03457280099391937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,32,0.014790399372577668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,16,0.010047999769449234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,8,0.00868159979581833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,4,0.008566399663686752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,2,0.00865280032157898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,1,0.006960000097751618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,128,0.07183039784431458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,64,0.04206719994544983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,32,0.021247999370098115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,16,0.014441600441932679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,8,0.008777599781751633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,4,0.008595199882984161
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,2,0.008579199761152267
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,1,0.007187200337648391
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,128,0.10094079971313477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,16,0.01525759994983673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,4,0.008700799942016602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,8,0.009862399846315383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,2,0.008684799820184708
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,1,0.007180800288915634
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,128,0.12603839635848998
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,64,0.06689599752426148
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,32,0.04060479998588562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,16,0.019596800208091736
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,8,0.014531199634075165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,4,0.00883520022034645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,2,0.008633600175380706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,1,0.007305599749088287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,128,0.1861631989479065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,64,0.09699199795722961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,32,0.05167040228843689
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,16,0.026800000667572023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,8,0.014643199741840363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,4,0.0100832000374794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,2,0.008787199854850769
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,1,0.007238399982452392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,64,0.12254079580307006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,128,0.24368000030517578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,32,0.06387839913368225
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,16,0.039852800965309146
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,4,0.013011200726032257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,2,0.008806400001049042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,8,0.021436800062656403
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,1,0.007420799881219864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,64,0.1787775993347168
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,128,0.349180793762207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,32,0.09264640212059021
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,16,0.05171520113945007
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,64,0.05429760217666626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,8,0.022137600183486938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,4,0.013199999928474426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,2,0.010019200295209885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,1,0.0084927998483181
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,32,0.027260801196098326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,32,0.12045760154724121
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,64,0.23260800838470458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,128,0.46672959327697755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,4,0.021155199408531188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,16,0.06557440161705017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,8,0.03912000060081482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,2,0.014777599275112152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,1,0.00894080027937889
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,32,0.17158080339431764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,64,0.34327359199523927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,8,0.05044159889221191
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,16,0.09318079948425292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,4,0.022067199647426605
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,1,0.010601600259542465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,128,0.7055263996124268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,2,0.015689599514007568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,32,0.23396480083465576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,64,0.4579455852508545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,16,0.1234015941619873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,8,0.06331200003623963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,4,0.03874239921569824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,2,0.021388800442218782
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,1,0.012931199371814727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,128,0.919491195678711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,32,0.35301759243011477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,16,0.17845439910888672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,8,0.08968960046768189
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,64,0.6828159809112548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,4,0.050672000646591185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,2,0.02237119972705841
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,1,0.013359999656677246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,128,1.3949088096618651
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,32,0.4585696220397949
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,16,0.23372480869293213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,64,0.9069472312927246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,8,0.11795200109481811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,4,0.06492159962654113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,2,0.03893119990825653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,1,0.019593599438667297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,128,1.8331647872924806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,16,0.2890496015548706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,64,1.113916778564453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,32,0.5653439998626709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,8,0.1473536014556885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,4,0.07651519775390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,2,0.042854401469230655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,1,0.020531199872493744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,128,2.302140808105469
