framework,version,device,op_name,kernel_source,bmm_dtype,num_tokens,num_heads,latency
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1,128,0.005846399813890457
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1,64,0.00469760000705719
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1,64,0.007596799731254577
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1,4,0.003923200070858002
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1,4,0.007295999675989151
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1,32,0.005209600180387497
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1,2,0.00424639992415905
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1,1,0.0038784001022577285
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1,16,0.0046720001846551895
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1,1,0.007360000163316727
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2,128,0.00599679984152317
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2,64,0.0048767998814582825
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1,8,0.005318399891257286
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2,32,0.0050016000866889955
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1,128,0.00836160033941269
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1,32,0.007440000027418137
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2,16,0.004681599885225296
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2,8,0.0038975998759269714
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2,64,0.008643200248479843
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2,4,0.003824000060558319
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2,4,0.007132799923419952
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2,2,0.004169600084424019
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2,2,0.0074592001736164095
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2,1,0.003782400116324425
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2,1,0.0071263998746871945
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4,128,0.005913599953055381
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4,128,0.009455999732017517
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4,64,0.0048191998153924945
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4,64,0.00851840004324913
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4,32,0.004332799836993218
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4,32,0.008249600231647492
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4,16,0.004038399830460548
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4,16,0.007699199765920639
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4,8,0.0038336001336574553
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4,8,0.007254400104284286
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4,4,0.0038143999874591826
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4,4,0.007481600344181061
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4,2,0.004291199892759323
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4,2,0.007852800190448761
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4,1,0.004383999854326248
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4,1,0.007135999947786331
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8,128,0.005849599838256836
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8,128,0.009718400239944459
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8,64,0.004687999933958053
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8,64,0.008499199897050858
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8,32,0.004220800101757049
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8,32,0.008172799646854401
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8,16,0.00398080013692379
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8,16,0.008028800040483475
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8,8,0.003961599990725517
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8,8,0.00764160007238388
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8,4,0.0038495998829603194
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1,8,0.006831999868154526
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8,4,0.0071552000939846035
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8,2,0.007187200337648391
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8,1,0.0039007999002933502
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2,16,0.007116799801588058
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8,1,0.007184000313282013
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1,2,0.007132799923419952
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2,8,0.0074592001736164095
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1,16,0.0070431999862194065
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,16,64,0.00483199991285801
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2,32,0.007209599763154983
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,16,16,0.005008000135421753
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,16,128,0.011612799763679505
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,16,8,0.003923200070858002
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,16,64,0.009657599776983262
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8,2,0.004992000013589859
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,16,4,0.003907199949026108
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,16,2,0.003948799893260002
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,16,128,0.005974400043487549
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,16,2,0.007513599842786789
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,16,16,0.007952000200748443
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,32,128,0.0063231997191905975
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,16,32,0.004335999861359597
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,32,64,0.005161599814891815
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,32,64,0.011385600268840789
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,32,32,0.0046016000211238865
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,32,32,0.009334400296211243
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,32,16,0.004342399910092354
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,32,16,0.008457600325345992
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,32,8,0.004239999875426293
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2,128,0.008595199882984161
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,32,8,0.007872000336647034
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,32,4,0.005158400163054466
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,32,4,0.007583999633789062
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,32,2,0.004953600093722343
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,32,1,0.004956800118088722
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,32,1,0.007731200009584427
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,48,128,0.007107199728488922
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,48,128,0.01838400065898895
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,48,64,0.005590400099754334
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,48,64,0.013193599879741669
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,48,32,0.0051231998950243
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,48,32,0.010080000013113022
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,16,8,0.007887999713420867
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,48,16,0.004899200052022934
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,16,32,0.008556800335645676
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,48,8,0.0050592001527547835
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,48,8,0.008284799754619598
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,48,4,0.005587200075387955
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,48,4,0.007552000135183335
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,48,2,0.004927999898791313
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,48,2,0.007497599720954895
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,48,1,0.004886399954557419
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,32,128,0.014864000678062438
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,48,1,0.0077344000339508055
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,64,128,0.007491199672222138
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,64,128,0.022047999501228332
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,64,64,0.005756799876689911
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,64,64,0.014892800152301789
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,64,32,0.005030399933457374
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,64,32,0.011376000195741653
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,64,16,0.008991999924182892
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,64,8,0.004841599985957146
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,64,8,0.008079999685287475
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,64,4,0.004982399940490723
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,64,4,0.008099199831485748
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,64,2,0.004969599843025208
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,32,2,0.007334399968385696
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,64,1,0.004112000018358231
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,64,1,0.007395199686288834
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,80,128,0.008124800026416778
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,80,128,0.02526400089263916
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,80,64,0.0061760000884532925
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,80,64,0.01669439971446991
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,80,32,0.005107200145721436
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,80,32,0.012652799487113953
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,48,16,0.008671999722719193
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,16,4,0.007039999961853028
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,80,16,0.009206400066614152
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,16,1,0.007340800017118454
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,80,8,0.008291199803352356
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,80,4,0.004678399860858917
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,80,4,0.007980799674987793
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,80,2,0.0038336001336574553
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,80,2,0.007849600166082382
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,80,1,0.007817599922418594
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,96,128,0.00854720026254654
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,64,16,0.0050432000309228895
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,96,64,0.007056000083684922
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,96,64,0.018060800433158875
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,96,32,0.005116799846291542
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,96,32,0.0123648002743721
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,96,16,0.00517439991235733
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,64,2,0.007654400169849395
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,96,16,0.009759999811649323
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,96,8,0.005036799982190132
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,96,8,0.008297599852085114
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,96,4,0.0049632001668214795
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,96,4,0.008169600367546081
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,96,2,0.007807999849319458
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,96,1,0.005027199909090996
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,96,1,0.007724799960851669
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,128,128,0.010080000013113022
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,128,128,0.035743999481201175
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,128,64,0.0072063997387886046
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,128,64,0.021414400637149812
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,16,1,0.003910399973392487
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,128,32,0.014630399644374847
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,128,16,0.004623999819159508
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,128,16,0.011033599823713302
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,128,8,0.004374400153756142
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,128,8,0.009244800359010697
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,96,128,0.028460800647735596
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,128,4,0.009350399672985076
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,128,2,0.0038975998759269714
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,128,1,0.003891199827194214
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,128,2,0.00788159966468811
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,128,1,0.007449600100517273
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,160,128,0.011760000139474869
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,160,64,0.007993599772453308
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,160,64,0.02499520033597946
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,160,32,0.005772799998521805
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,160,32,0.016041600704193117
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,160,16,0.004822399839758873
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,96,2,0.004982399940490723
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,160,16,0.011711999773979187
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,160,8,0.009708800166845322
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,160,4,0.0042015999555587765
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,160,4,0.008617600053548813
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,160,2,0.004092799872159958
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,160,2,0.00801599994301796
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,160,1,0.004016000032424927
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,128,32,0.00535999983549118
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,192,128,0.01629119962453842
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,192,128,0.05398079752922058
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,80,1,0.003961599990725517
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,192,64,0.008099199831485748
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,192,32,0.006115199998021126
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,192,32,0.017824000120162962
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,192,16,0.004956800118088722
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,128,4,0.004073600098490715
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,192,16,0.012297599762678146
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,192,8,0.004588799923658371
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,192,8,0.009679999947547913
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,192,4,0.004294399917125702
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,192,4,0.008303999900817871
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,192,2,0.003984000161290168
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,192,2,0.008032000064849854
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,192,1,0.003888000175356865
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,160,128,0.04259839951992035
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,192,1,0.007571200281381607
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,256,128,0.02200320065021515
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,256,64,0.009510400146245957
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,256,128,0.06876159906387329
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,256,64,0.03568640053272247
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,256,32,0.021385599672794343
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,256,16,0.005318399891257286
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,160,8,0.004950400069355964
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,256,16,0.014441600441932679
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,256,8,0.005203200131654739
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,256,8,0.010921599715948105
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,256,4,0.00493439994752407
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,256,4,0.008736000210046769
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,256,2,0.004924799874424934
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,256,2,0.007926400005817413
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,256,1,0.004985599964857102
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,256,1,0.008022399991750718
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,320,128,0.023596799373626708
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,80,16,0.004569600149989128
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,320,64,0.012185599654912949
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,320,128,0.08280959725379944
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,320,64,0.04451520144939423
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,320,16,0.005696000158786773
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,320,16,0.015987199544906617
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,320,8,0.004806400090456009
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,320,8,0.011699199676513672
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,320,4,0.00448639988899231
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,320,4,0.009753599762916565
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,320,2,0.004092799872159958
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,320,2,0.008303999900817871
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,320,1,0.004083200171589851
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,320,1,0.007705599814653396
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,384,128,0.027449598908424376
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,384,128,0.09969599843025208
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,256,32,0.006828799843788147
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,384,64,0.05222079753875732
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,384,32,0.00827839970588684
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,384,32,0.02852480113506317
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,384,16,0.0062015999108552934
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,384,16,0.01793919950723648
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,384,8,0.004924799874424934
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,384,8,0.012617599964141846
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,384,4,0.004492799937725067
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,320,32,0.0071103997528553006
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,320,32,0.024454399943351746
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,384,2,0.004931199923157692
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,384,1,0.004927999898791313
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,384,1,0.008300799876451492
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,512,128,0.03362239897251129
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,192,64,0.028003200888633728
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,512,128,0.12386239767074585
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,512,64,0.06807680130004883
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,512,32,0.00968639999628067
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,512,32,0.03530240058898926
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,512,16,0.006857600063085556
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,512,16,0.021212799847126006
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,512,8,0.005411199852824211
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,512,8,0.014207999408245086
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,512,4,0.004816000163555145
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,512,4,0.01085439994931221
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,512,2,0.00493439994752407
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,512,2,0.009200000017881394
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,384,64,0.014297600090503692
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,512,1,0.005071999877691269
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,768,128,0.04512960016727448
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,768,128,0.1831936001777649
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,768,64,0.025510400533676147
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,768,64,0.0992031991481781
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,768,32,0.013910399377346038
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,768,16,0.008329600095748901
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,768,16,0.028751999139785767
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,768,8,0.006076800078153611
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,768,8,0.017871999740600587
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,768,4,0.0050016000866889955
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,384,4,0.010102400183677673
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,768,4,0.01239359974861145
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,768,2,0.004425600171089172
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,768,2,0.009724800288677216
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,768,1,0.004102399945259095
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1024,128,0.05600960254669189
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1024,128,0.24058239459991454
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1024,64,0.0316895991563797
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,512,64,0.018947200477123262
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1024,64,0.12427200078964233
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1024,32,0.01860159933567047
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1024,16,0.009433600306510925
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1024,16,0.035123199224472046
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1024,8,0.006809599697589874
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1024,8,0.021513600647449494
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1024,4,0.0053727999329566956
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1024,4,0.014521600306034088
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1024,2,0.004598399996757508
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1024,2,0.011075200140476226
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1024,1,0.004236799851059914
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1024,1,0.008956799656152726
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1536,128,0.0780672013759613
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,768,32,0.05212479829788208
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1536,64,0.043808001279830935
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,80,8,0.005270399898290634
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,384,2,0.008252800256013871
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,768,1,0.008342400193214417
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1536,64,0.1826688051223755
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1536,32,0.024784000217914583
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1536,32,0.09865599870681763
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1536,8,0.008211199939250947
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1536,8,0.028508800268173217
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1536,4,0.00605119988322258
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1536,4,0.017897599935531618
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1536,2,0.004944000020623207
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1024,32,0.0676576018333435
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1536,2,0.012240000069141388
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1536,1,0.004540799930691719
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1536,1,0.00989760011434555
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2048,128,0.10120960474014282
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2048,64,0.05485759973526001
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2048,128,0.47122559547424314
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2048,64,0.240067195892334
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2048,32,0.03115200102329254
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2048,32,0.12350080013275147
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2048,16,0.0673695981502533
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2048,8,0.009523200243711472
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,512,1,0.008352000266313553
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2048,4,0.007023999840021134
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2048,4,0.021619200706481934
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2048,2,0.005478399991989136
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1536,128,0.3547327995300293
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2048,2,0.014537599682807923
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2048,1,0.01096320003271103
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,3072,128,0.14459840059280396
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,160,1,0.007542400062084198
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,3072,64,0.07654399871826172
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,3072,128,0.707583999633789
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,1536,16,0.013612799346446991
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,3072,64,0.35530240535736085
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,3072,32,0.18301119804382324
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,3072,16,0.09899200201034546
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,3072,8,0.013849599659442902
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,3072,8,0.05200319886207581
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,3072,4,0.008287999778985977
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,3072,4,0.028297600150108338
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,3072,2,0.006019200012087822
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,3072,2,0.017667199671268462
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2048,16,0.01721920073032379
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,3072,1,0.005311999842524529
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,3072,1,0.01266240030527115
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4096,128,0.19052480459213256
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,2048,8,0.034985598921775815
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4096,64,0.09857280254364013
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4096,32,0.053439998626708986
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4096,128,0.9475168228149414
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4096,32,0.2398848056793213
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4096,16,0.030371201038360596
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4096,16,0.12384959459304809
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,2048,1,0.0044895999133586885
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4096,8,0.06728000044822693
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4096,4,0.009379199892282485
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4096,2,0.006960000097751618
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4096,4,0.035359999537467955
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4096,2,0.021542400121688843
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4096,1,0.005395200103521347
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4096,1,0.014364799857139588
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,6144,128,0.2808703899383545
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,6144,64,0.14690879583358765
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,1536,16,0.0520799994468689
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,3072,32,0.04188799858093262
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,6144,32,0.07512959837913513
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,3072,16,0.024400000274181367
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,6144,16,0.041526401042938234
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,6144,128,1.4294719696044922
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,6144,32,0.35539200305938723
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,6144,64,0.710368013381958
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,6144,16,0.18297599554061889
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,6144,4,0.013631999492645264
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,6144,2,0.008403199911117553
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,6144,4,0.050911998748779295
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,6144,1,0.006102399900555611
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,6144,1,0.01759359985589981
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8192,128,0.36893439292907715
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8192,64,0.20142719745635987
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8192,32,0.09825599789619446
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8192,64,0.9542112350463867
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8192,16,0.05331519842147827
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8192,128,1.93591365814209
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8192,16,0.23911359310150146
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,4096,64,0.4732704162597656
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8192,8,0.12367039918899536
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,4096,8,0.018681600689888
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8192,2,0.009603200107812881
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8192,2,0.035471999645233156
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8192,1,0.006886400282382965
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8192,1,0.021084800362586975
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,6144,8,0.02389120012521744
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,6144,8,0.09888319969177246
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8192,32,0.4666816234588623
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8192,8,0.032815998792648314
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,float16,8192,4,0.016976000368595125
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,8192,4,0.0673632025718689
SGLang,0.5.9,NVIDIA H200,mla_gen_pre,default,fp8,6144,2,0.028358399868011475
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1,128,0.005382400006055832
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1,128,0.008057600259780884
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1,64,0.007648000121116638
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1,32,0.004566400125622749
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1,32,0.008099199831485748
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1,16,0.004416000097990036
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1,64,0.005129599943757057
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1,16,0.007862400263547897
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1,8,0.005932800099253654
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1,4,0.004323200136423111
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1,8,0.008640000224113464
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1,2,0.004355200007557869
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1,1,0.004287999868392944
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1,2,0.0075103998184204105
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2,128,0.00835840031504631
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2,64,0.008172799646854401
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2,32,0.004515200108289719
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2,32,0.007795199751853943
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2,16,0.005062400177121162
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1,1,0.007548800110816956
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2,16,0.007987199723720551
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2,8,0.008825600147247314
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2,4,0.004425600171089172
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2,4,0.00769599974155426
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2,2,0.004268800094723702
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2,2,0.0073183998465538025
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2,1,0.004208000004291534
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2,1,0.0073472000658512115
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4,128,0.0055808000266551975
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4,128,0.009225600212812424
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4,64,0.004944000020623207
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4,32,0.0046016000211238865
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4,32,0.00820159986615181
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4,16,0.004156799986958504
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4,16,0.0075552001595497135
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4,8,0.004758400097489357
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4,4,0.007577600330114365
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1,4,0.006921599805355072
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4,2,0.0068000003695487974
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2,8,0.005903999879956245
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4,1,0.007878399640321731
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8,128,0.005859199911355972
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8,128,0.009916800260543823
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8,64,0.004860800132155419
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2,64,0.004956800118088722
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2,128,0.00549440011382103
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8,32,0.004335999861359597
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8,16,0.004176000133156776
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4,1,0.004128000140190125
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8,8,0.004227200150489807
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8,8,0.007999999821186066
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8,4,0.0042975999414920805
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8,4,0.007977599650621415
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8,2,0.00419199988245964
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8,2,0.008675199747085572
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8,1,0.004086399823427201
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8,1,0.007264000177383423
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16,128,0.00565119981765747
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4,2,0.004172800108790398
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16,64,0.0046847999095916745
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4,4,0.004608000069856644
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16,32,0.004444799944758415
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16,128,0.011321599781513213
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8,64,0.008825600147247314
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16,16,0.004204799979925155
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16,8,0.004188799858093261
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16,64,0.009312000125646591
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16,16,0.008076799660921096
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16,8,0.00759039968252182
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16,4,0.0043519999831914905
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16,2,0.00785600021481514
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16,1,0.004211200028657913
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16,1,0.007750400155782699
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,32,128,0.006092799827456474
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,32,128,0.014403200149536133
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,32,64,0.004915200173854828
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,32,64,0.010822399705648422
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,32,32,0.0045855998992919925
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,32,32,0.008950400352478027
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,32,16,0.004495999962091446
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,32,16,0.008441600203514098
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,32,8,0.004371200129389763
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,32,8,0.008249600231647492
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,32,4,0.004364800080657006
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,32,4,0.007782399654388428
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,32,2,0.0042975999414920805
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,32,2,0.007532799988985062
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,32,1,0.004383999854326248
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,32,1,0.007849600166082382
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,48,128,0.006543999910354615
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,48,128,0.01785600036382675
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,48,64,0.005241600051522255
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,48,64,0.01276479959487915
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,48,32,0.004704000055789947
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,48,32,0.009734400361776353
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,48,16,0.004534399881958961
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16,2,0.004198399931192398
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,48,8,0.004495999962091446
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,48,8,0.008416000008583068
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,48,4,0.004287999868392944
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4,64,0.007625599950551986
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,48,4,0.007507199794054032
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4,8,0.007980799674987793
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16,4,0.007628799974918365
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,48,2,0.00735040009021759
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,48,1,0.0073632001876831055
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,48,2,0.005369599908590317
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,64,128,0.02099200040102005
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16,32,0.008131200075149536
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,64,64,0.005488000065088272
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,64,64,0.014192000031471252
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,64,32,0.005056000128388405
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,64,32,0.01125440001487732
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,64,16,0.0048767998814582825
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,64,8,0.004979199916124344
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,64,8,0.008675199747085572
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,64,4,0.0050016000866889955
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,64,4,0.00809279978275299
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,64,2,0.005004800111055374
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,64,2,0.007811199873685837
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,48,16,0.008614400029182434
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,64,1,0.004838399961590767
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,64,1,0.00756480023264885
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,80,128,0.00732479989528656
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8,32,0.007481600344181061
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,80,64,0.005488000065088272
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,80,64,0.015523199737071992
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,80,32,0.004639999940991402
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,80,16,0.005728000029921532
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,80,32,0.01170239970088005
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,80,8,0.0050016000866889955
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,80,16,0.00974079966545105
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,80,8,0.008924800157546996
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,80,2,0.004377600178122521
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,80,4,0.00833280012011528
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,80,2,0.00827839970588684
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,80,1,0.004252799972891808
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,80,1,0.00801599994301796
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,96,128,0.007747200131416321
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,96,128,0.027695998549461365
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,96,64,0.005737600103020668
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,96,64,0.017078399658203125
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,96,32,0.004899200052022934
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,96,32,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,96,16,0.004710400104522705
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,96,16,0.010255999863147736
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,96,8,0.00865280032157898
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,96,4,0.004287999868392944
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,96,4,0.007884799689054488
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,96,2,0.004303999990224838
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,96,2,0.007644800096750259
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,96,1,0.0043136000633239744
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,96,1,0.00782719999551773
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,128,128,0.008607999980449676
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,64,128,0.006892800331115723
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,128,128,0.03506560027599335
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,128,64,0.020656000077724456
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,128,32,0.005023999884724617
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,64,16,0.009427200257778167
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,128,32,0.014233599603176116
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,128,16,0.00493439994752407
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,128,16,0.010659199953079224
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,128,8,0.00907519981265068
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,128,4,0.004527999833226204
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,128,4,0.008131200075149536
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,128,2,0.004236799851059914
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,128,2,0.008054400235414505
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,80,128,0.02417919933795929
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,128,1,0.008211199939250947
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,160,128,0.009369599819183349
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,160,128,0.04305280148983002
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,160,64,0.006553599983453751
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,160,64,0.0235167995095253
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,160,32,0.00517439991235733
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,160,32,0.015833599865436553
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,160,16,0.004793599992990494
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,96,8,0.004534399881958961
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,160,16,0.011587200313806533
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,160,8,0.004720000177621841
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,160,8,0.00936639979481697
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,160,4,0.004387199878692627
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,160,2,0.004329600185155868
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,160,2,0.00804160013794899
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,160,1,0.004163200035691261
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,160,1,0.007846400141716003
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,192,128,0.013446399569511413
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,128,64,0.0063008002936840056
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,192,128,0.05119040012359619
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,192,64,0.00703359991312027
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,192,64,0.02682879865169525
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,192,32,0.0054687999188899996
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,192,32,0.017312000691890716
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,192,16,0.012716799974441528
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,192,8,0.004518400132656098
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,192,8,0.010000000149011612
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,192,4,0.004441599920392036
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8,16,0.0079584002494812
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,192,4,0.008601599931716919
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,128,1,0.004361600056290627
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,192,2,0.007996799796819687
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,192,1,0.00432640016078949
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,192,1,0.008057600259780884
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,256,128,0.018540799617767334
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,256,128,0.06568959951400757
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,256,64,0.007676800340414047
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,256,64,0.034246399998664856
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,256,32,0.005830400064587593
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,256,32,0.02011519968509674
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,256,16,0.004921599850058555
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,256,16,0.01385280042886734
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,256,8,0.004675199836492538
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,256,8,0.010873600095510482
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,256,4,0.0045855998992919925
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,256,4,0.00939520001411438
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,160,4,0.00830719992518425
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,256,2,0.004604800045490265
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,256,1,0.004310400038957596
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,256,2,0.00833280012011528
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,256,1,0.008291199803352356
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,320,128,0.02159679979085922
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,320,128,0.0806111991405487
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,320,64,0.04212799966335297
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,320,32,0.006454399973154068
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,320,32,0.02343679964542389
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,192,16,0.004979199916124344
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,128,8,0.004704000055789947
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,320,16,0.015849600732326507
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,320,8,0.005046400055289268
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,320,8,0.011884800344705581
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,320,4,0.00549440011382103
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,320,4,0.009548799693584442
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,320,2,0.004479999840259552
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,320,2,0.00854720026254654
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,320,1,0.008019199967384339
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,192,2,0.004425600171089172
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,384,128,0.024563199281692503
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,384,64,0.009382399916648864
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,80,4,0.004892800003290176
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,384,128,0.09683840274810791
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,384,64,0.05023679733276367
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,384,32,0.006675200164318084
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,384,32,0.02699199914932251
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,384,16,0.017475199699401856
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,384,8,0.004912000149488449
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,384,4,0.004614400118589402
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,384,4,0.010182400047779084
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,384,2,0.004467200115323066
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,384,2,0.008656000345945358
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,384,1,0.004291199892759323
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,384,1,0.0081216000020504
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,512,128,0.030041599273681642
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,512,128,0.12566720247268676
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,512,64,0.014646400511264802
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,512,64,0.06529600024223328
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,320,64,0.00836160033941269
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,512,32,0.007731200009584427
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,512,32,0.033616000413894655
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,512,16,0.020572799444198608
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,512,8,0.005020799860358238
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,512,8,0.014134399592876434
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,512,4,0.005209600180387497
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,512,4,0.010889600217342376
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,512,2,0.004467200115323066
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,320,1,0.004492799937725067
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,512,1,0.00456320010125637
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,512,1,0.008326400071382523
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,768,128,0.0411327987909317
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,768,64,0.02228800058364868
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,768,128,0.1851871967315674
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,768,32,0.009558399766683578
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,768,64,0.0957759976387024
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,768,32,0.04958080053329468
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,768,16,0.006547199934720993
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,768,8,0.005318399891257286
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,768,16,0.026812800765037538
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,768,8,0.017020800709724428
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,384,8,0.012863999605178833
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,768,4,0.004841599985957146
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,768,4,0.012227199971675873
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,768,2,0.005315199866890908
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,768,1,0.004323200136423111
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,768,1,0.00896959975361824
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1024,128,0.05216320157051087
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1024,64,0.028336000442504884
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1024,128,0.24270401000976563
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1024,64,0.12407679557800293
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1024,32,0.01149120032787323
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1024,16,0.007724799960851669
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1024,16,0.03380799889564514
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1024,8,0.005615999922156334
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1024,8,0.020547200739383698
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,512,16,0.00565119981765747
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1024,4,0.004985599964857102
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1024,2,0.004992000013589859
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1024,4,0.014259199798107147
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,320,16,0.00514880008995533
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1024,1,0.004598399996757508
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1024,1,0.00897279977798462
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1536,128,0.07394880056381226
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1536,64,0.04041920006275177
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1536,128,0.3617952108383179
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,48,1,0.00432640016078949
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,512,2,0.009033600240945816
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1536,32,0.021427200734615327
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1536,16,0.009312000125646591
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1536,32,0.0951903998851776
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1536,16,0.049235200881958006
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1536,8,0.006355199962854385
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1536,4,0.005206400156021118
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,384,16,0.005571199953556061
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1536,4,0.017212800681591034
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1536,2,0.004937599971890449
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1536,2,0.012652799487113953
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,1536,1,0.0047391999512910845
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1536,1,0.009990400075912476
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2048,128,0.09604160189628601
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2048,64,0.05091519951820374
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2048,128,0.4880064010620117
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2048,64,0.24178240299224854
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2048,32,0.1246783971786499
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1024,32,0.06483839750289917
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2048,16,0.011311999708414077
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2048,16,0.06495680212974549
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2048,8,0.007875200361013412
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2048,4,0.005606399849057198
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2048,4,0.020339199900627138
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2048,2,0.0050016000866889955
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2048,2,0.01417279988527298
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2048,1,0.0048767998814582825
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1024,2,0.011129599809646607
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2048,1,0.01128000020980835
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,3072,128,0.13885120153427125
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,3072,64,0.07057279944419861
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,3072,128,0.751203203201294
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,3072,64,0.35732159614562986
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,3072,32,0.03961920142173767
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,3072,16,0.020803199708461763
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,3072,32,0.18451839685440063
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,3072,16,0.09468799829483032
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,3072,8,0.009270399808883667
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,3072,4,0.006364800035953522
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1536,8,0.02683520019054413
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,3072,4,0.026627200841903686
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,3072,2,0.017484800517559053
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,3072,1,0.005036799982190132
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,3072,1,0.012720000743865967
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,768,2,0.010252799838781357
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4096,128,0.19292160272598266
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4096,64,0.09135680198669434
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,2048,32,0.02729920148849487
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4096,32,0.04935680031776428
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4096,128,1.0241439819335938
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4096,32,0.24165759086608887
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4096,16,0.028428798913955687
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4096,16,0.12404160499572754
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4096,8,0.011961600184440613
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4096,8,0.06509439945220948
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4096,4,0.008012799918651581
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4096,2,0.005561599880456925
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4096,2,0.020553599298000335
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,4096,1,0.004915200173854828
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4096,1,0.014032000303268432
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,6144,128,0.2785696029663086
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,6144,64,0.13130559921264648
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,1536,64,0.18435200452804565
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,6144,128,1.5715200424194335
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,6144,64,0.7544896125793457
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,6144,16,0.038515201210975646
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,6144,32,0.3585855960845947
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,6144,16,0.18369280099868773
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,6144,8,0.0945855975151062
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,3072,2,0.005392000079154968
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,6144,4,0.009107200056314468
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,6144,2,0.0063231997191905975
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,6144,2,0.026915198564529418
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,6144,1,0.005145600065588951
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,6144,1,0.0169855996966362
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4096,64,0.4790976047515869
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8192,128,0.38248319625854493
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8192,64,0.1693824052810669
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,2048,8,0.03362880051136017
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8192,32,0.08948799967765808
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8192,64,1.027132797241211
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8192,16,0.04806720018386841
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8192,128,2.089814376831055
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8192,16,0.24038081169128417
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8192,8,0.028537601232528687
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8192,8,0.12376639842987061
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8192,4,0.06493120193481446
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8192,2,0.007577600330114365
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8192,2,0.033667200803756715
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,6144,32,0.06935999989509582
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8192,1,0.02036159932613373
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,3072,8,0.0489984005689621
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,6144,8,0.02112639993429184
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,12288,64,0.24936320781707763
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,12288,128,0.5553120136260986
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,12288,32,0.1401695966720581
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,12288,32,0.7583903789520263
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,12288,64,1.527302360534668
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,12288,16,0.06854720115661621
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,12288,16,0.359279990196228
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,12288,8,0.038950398564338684
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,12288,128,3.1533151626586915
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,12288,8,0.18366400003433228
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,4096,4,0.03351039886474609
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,12288,4,0.0222463995218277
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,12288,4,0.09453439712524414
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,12288,2,0.04916479885578155
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,12288,1,0.006540799885988236
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,8192,32,0.47813758850097654
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,12288,1,0.027030399441719054
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16384,64,0.32952959537506105
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8192,4,0.011497599631547928
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16384,128,0.7322336196899414
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16384,32,0.16868480443954467
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,8192,1,0.005612799897789955
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16384,16,0.4787168025970459
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16384,32,1.0235584259033204
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16384,8,0.04838719964027405
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16384,4,0.0265855997800827
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16384,64,2.0340511322021486
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,6144,4,0.0496832013130188
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16384,2,0.011296000331640244
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16384,1,0.007711999863386154
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16384,1,0.03351039886474609
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16384,128,4.208755111694336
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,20480,128,0.9266688346862793
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,20480,64,0.40766081809997556
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,16384,16,0.09604480266571044
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,20480,32,0.20736958980560302
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,20480,64,2.55861759185791
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,20480,16,0.11535680294036865
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16384,8,0.24103999137878418
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,20480,8,0.05828160047531128
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,20480,128,5.273958587646485
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16384,2,0.06480960249900818
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,20480,16,0.6161503791809082
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,20480,8,0.29935359954833984
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,20480,32,1.2884896278381348
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,20480,4,0.15387200117111205
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,20480,2,0.0786624014377594
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,20480,1,0.008284799754619598
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,12288,2,0.009110400080680847
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,16384,4,0.12357120513916016
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,20480,4,0.034201601147651674
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,float16,20480,2,0.017180800437927246
SGLang,0.5.9,NVIDIA H200,mla_gen_post,default,fp8,20480,1,0.04071359932422638
