framework,version,device,op_name,kernel_source,bmm_dtype,num_tokens,num_heads,latency
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,128,0.01228799968957901
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,64,0.007884799689054488
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,16,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,8,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,4,0.006348799914121628
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,2,0.006348799914121628
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,1,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,128,0.01443839967250824
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,64,0.010547199845314026
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,32,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,16,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,8,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,4,0.006758400052785873
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,2,0.006348799914121628
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2,1,0.006348799914121628
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,128,0.014233599603176116
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,32,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,16,0.006348799914121628
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,8,0.006246399879455566
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,2,0.007680000364780426
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,1,0.006246399879455566
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,128,0.01454080045223236
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,64,0.010547199845314026
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,32,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,16,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1,32,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,8,0.007577600330114365
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,4,0.006963200122117996
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,2,0.006963200122117996
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8,1,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,128,0.014847999811172486
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,64,0.010649599879980088
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,32,0.007475200295448303
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,16,0.006656000018119812
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,8,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,2,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,1,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,128,0.015667200088500977
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,64,0.010956799983978272
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,32,0.007577600330114365
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,16,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,8,0.006348799914121628
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,4,0.006656000018119812
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,2,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,32,1,0.006143999844789505
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,128,0.016383999586105348
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,64,0.011366400122642516
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,32,0.010444799810647965
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,64,0.010751999914646149
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,16,0.007475200295448303
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,8,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,4,0.006143999844789505
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,2,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,48,1,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,128,0.017510400712490083
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,64,0.011264000087976456
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,32,0.010444799810647965
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,16,0.007680000364780426
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,8,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,4,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,2,0.006348799914121628
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4,4,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,64,1,0.008191999793052674
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,128,0.023244799673557283
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,64,0.016281600296497344
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,32,0.013209599256515502
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,16,0.009113600105047226
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,8,0.006656000018119812
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,4,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,2,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,80,1,0.006348799914121628
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,128,0.024063999950885772
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,64,0.016281600296497344
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,32,0.013516800105571746
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,16,0.009216000139713288
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,16,4,0.006758400052785873
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,8,0.006656000018119812
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,4,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,2,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,96,1,0.006348799914121628
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,128,0.03041279911994934
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,64,0.01730560064315796
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,32,0.016281600296497344
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,16,0.010342399775981902
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,8,0.007577600330114365
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,4,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,2,0.006246399879455566
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,128,1,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,128,0.050892800092697144
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,64,0.022835199534893037
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,32,0.015564799308776855
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,16,0.011776000261306763
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,8,0.008089599758386612
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,4,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,2,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,160,1,0.006656000018119812
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,64,0.023449599742889404
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,32,0.015872000157833098
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,16,0.011776000261306763
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,8,0.009216000139713288
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,4,0.006758400052785873
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,2,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,1,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,64,0.02447360008955002
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,16,0.011878400295972823
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,8,0.010547199845314026
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,4,0.007475200295448303
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,2,0.006656000018119812
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,1,0.006656000018119812
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,128,0.08857600092887878
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,64,0.03307519853115082
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,32,0.019046400487422944
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,192,128,0.054579198360443115
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,16,0.012185599654912949
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,8,0.012083200365304947
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,4,0.008191999793052674
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,2,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,320,1,0.006656000018119812
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,128,0.09256960153579712
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,64,0.03604480028152466
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,32,0.019251200556755065
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,16,0.011980800330638886
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,8,0.013619199395179749
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,4,0.009318400174379349
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,2,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,384,1,0.006656000018119812
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,128,0.05928959846496582
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,128,0.10240000486373901
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,64,0.04730879962444305
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,32,0.02396160066127777
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,16,0.016076800227165223
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,8,0.016076800227165223
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,4,0.010547199845314026
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,2,0.008294399827718735
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,256,32,0.01648640036582947
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,512,1,0.007475200295448303
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,128,0.14929920434951782
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,64,0.0681984007358551
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,32,0.034815999865531924
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,16,0.01935359984636307
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,8,0.013619199395179749
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,4,0.008703999966382981
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,2,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,768,1,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,64,0.08611840009689331
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,128,0.1912832021713257
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,32,0.04444159865379334
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,16,0.023756800591945647
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,8,0.01730560064315796
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,4,0.011468800157308579
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,2,0.008396799862384795
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1024,1,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,64,0.12410880327224731
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,128,0.2846719980239868
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,32,0.06492159962654113
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,16,0.03328000009059906
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,8,0.022732800245285033
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,2,0.008703999966382981
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,4,0.013926400244235993
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,1536,1,0.007577600330114365
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,64,0.1597440004348755
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,128,0.37068800926208495
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,32,0.0831488013267517
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,16,0.04352000057697296
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,8,0.02815999984741211
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,4,0.017510400712490083
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,2,0.011878400295972823
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,2048,1,0.008089599758386612
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,64,0.23603200912475586
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,32,0.12031999826431275
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,16,0.06307839751243591
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,8,0.040243199467658995
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,128,0.5516287803649902
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,4,0.022835199534893037
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,2,0.013926400244235993
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,3072,1,0.008703999966382981
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,16,0.08232960104942322
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,32,0.15708160400390625
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,64,0.3097599983215332
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,8,0.05242879986763001
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,4,0.02805759906768799
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,2,0.017715199291706084
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,1,0.011468800157308579
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,4096,128,0.7317503929138184
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,32,0.23244800567626953
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,16,0.12042239904403687
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,8,0.06113280057907104
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,64,0.45946879386901857
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,4,0.03358719944953918
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,2,0.01945600062608719
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,1,0.011776000261306763
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,6144,128,0.9142271995544433
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,8,0.08232960104942322
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,16,0.1563647985458374
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,32,0.30617599487304686
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,4,0.04167680144309997
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,64,0.6098944187164307
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,1,0.019968000054359437
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,2,0.02396160066127777
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_pre,default,float16,8192,128,1.218560028076172
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,128,0.011776000261306763
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,32,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,16,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,8,0.006553599983453751
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,4,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,2,0.006656000018119812
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,1,0.006451199948787689
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,128,0.012595200538635254
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,64,0.009625600278377533
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,32,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,16,0.006963200122117996
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,8,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,4,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,2,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2,1,0.006758400052785873
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,128,0.012595200538635254
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,64,0.009523200243711472
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,32,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,16,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,4,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,128,0.012902399897575379
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,64,0.009625600278377533
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,32,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,16,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,4,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,2,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,1,0.006860800087451935
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1,64,0.009216000139713288
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,128,0.012800000607967377
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,64,0.009728000313043595
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,32,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,16,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,8,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,4,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,2,0.006963200122117996
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16,1,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,128,0.013107199966907502
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,64,0.010137599706649781
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,32,0.007475200295448303
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,16,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,8,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,4,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,2,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,1,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8,8,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,32,1,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,128,0.013312000036239623
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,64,0.010342399775981902
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,32,0.007475200295448303
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,2,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4,8,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,8,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,4,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,2,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,1,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,128,0.013619199395179749
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,64,0.01085439994931221
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,32,0.007884799689054488
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,16,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,8,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,4,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,2,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,64,1,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,128,0.02109439969062805
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,64,0.0130048006772995
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,32,0.012800000607967377
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,16,0.007475200295448303
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,8,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,4,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,2,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,80,1,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,128,0.02815999984741211
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,48,16,0.007475200295448303
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,64,0.014336000382900237
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,32,0.009728000313043595
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,16,0.007577600330114365
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,8,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,4,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,2,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,96,1,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,128,0.03338240087032318
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,64,0.014643199741840363
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,16,0.007987199723720551
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,8,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,4,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,2,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,1,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,128,0.037376001477241516
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,64,0.02109439969062805
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,32,0.012800000607967377
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,16,0.008396799862384795
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,8,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,4,0.007168000191450119
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,2,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,160,1,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,128,0.040243199467658995
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,64,0.021196800470352172
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,32,0.0130048006772995
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,16,0.008703999966382981
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,8,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,4,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,2,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,192,1,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,128,0.045875200629234315
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,64,0.02181120067834854
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,32,0.013414399325847625
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,16,0.008908800035715102
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,8,0.008703999966382981
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,4,0.008703999966382981
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,2,0.008703999966382981
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,256,1,0.007065600156784058
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,128,0.05283839702606201
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,64,0.028569599986076354
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,32,0.013414399325847625
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,16,0.009625600278377533
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,8,0.008703999966382981
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,4,0.008703999966382981
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,2,0.008806400001049042
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,320,1,0.007577600330114365
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,128,0.05928959846496582
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,64,0.034406399726867674
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,32,0.014950400590896607
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,16,0.009830400347709656
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,128,32,0.009011200070381165
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,8,0.008908800035715102
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,4,0.008703999966382981
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,2,0.008601599931716919
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,384,1,0.006963200122117996
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,128,0.07178239822387696
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,64,0.04229120016098022
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,32,0.02160640060901642
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,16,0.01443839967250824
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,8,0.009011200070381165
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,4,0.008806400001049042
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,2,0.008806400001049042
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,512,1,0.007372800260782242
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,128,0.10147839784622192
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,64,0.053452801704406736
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,32,0.026931199431419372
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,16,0.015769599378108977
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,8,0.009932799637317658
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,2,0.008806400001049042
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,4,0.009011200070381165
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,768,1,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,128,0.12738560438156127
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,64,0.06717439889907836
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,32,0.03973119854927063
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,16,0.01966080069541931
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,8,0.014643199741840363
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,4,0.009830400347709656
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,2,0.008806400001049042
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1024,1,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,64,0.09410560131072998
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,128,0.18360320329666138
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,32,0.051507198810577394
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,16,0.026726400852203368
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,8,0.014847999811172486
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,4,0.010239999741315842
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,2,0.008806400001049042
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,1536,1,0.00727040022611618
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,64,0.12154879570007324
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,32,0.06297600269317627
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,128,0.2352128028869629
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,16,0.039526399970054624
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,8,0.021196800470352172
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,2,0.008806400001049042
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,4,0.012800000607967377
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,2048,1,0.007577600330114365
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,64,0.17715200185775756
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,32,0.09502720236778259
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,16,0.051609599590301515
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,128,0.3507200002670288
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,8,0.02242559939622879
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,2,0.010035199671983719
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,4,0.013414399325847625
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,3072,1,0.008601599931716919
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,32,0.118886399269104
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,64,0.2323456048965454
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,16,0.06543359756469727
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,8,0.03891200125217438
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,128,0.46940159797668457
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,4,0.021299199759960176
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,2,0.01515520066022873
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,4096,1,0.008908800035715102
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,64,0.3429375886917114
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,32,0.17428480386734008
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,16,0.09390079975128174
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,8,0.05140479803085327
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,4,0.02181120067834854
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,2,0.015667200088500977
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,1,0.010342399775981902
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,6144,128,0.6897664070129395
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,32,0.2333695888519287
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,64,0.4439040184020996
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,8,0.06338559985160827
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,16,0.12339199781417846
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,4,0.03880960047245026
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,1,0.013107199966907502
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,2,0.02232320010662079
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,8192,128,0.900607967376709
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,32,0.3379199981689453
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,16,0.17786879539489747
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,64,0.6807551860809327
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,4,0.051507198810577394
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,8,0.08970239758491516
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,2,0.02252800017595291
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,1,0.013312000036239623
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,12288,128,1.3609984397888184
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,32,0.45946879386901857
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,64,0.901529598236084
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,8,0.11816960573196411
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,16,0.23070719242095947
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,4,0.06389759778976441
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,2,0.03891200125217438
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,1,0.01966080069541931
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,16384,128,1.827225685119629
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,64,1.125068759918213
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,16,0.29757440090179443
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,32,0.5601280212402344
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,8,0.14428160190582276
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,4,0.07772160172462464
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,2,0.04249599874019623
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,1,0.020479999482631683
SGLang,0.5.8,NVIDIA A100-SXM4-80GB,mla_gen_post,default,float16,20480,128,2.275020790100098
