framework,version,device,op_name,kernel_source,bmm_dtype,num_tokens,num_heads,latency
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1,128,0.006233600154519081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,128,0.009151999652385712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1,64,0.004992000013589859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,64,0.00873280018568039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1,32,0.004508800059556961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,32,0.008035200089216233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1,16,0.004224000126123428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,16,0.007788799703121185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1,8,0.004128000140190125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,8,0.007680000364780426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1,4,0.004076800122857094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,4,0.007798399776220322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,2,0.008144000172615051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1,1,0.004172800108790398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1,1,0.007760000228881836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2,128,0.006451199948787689
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1,2,0.0049183998256921765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2,8,0.004150399938225746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2,64,0.005020799860358238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2,32,0.004844800010323525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2,16,0.004518400132656098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2,2,0.004873599857091904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,128,0.009440000355243682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2,1,0.0040895998477935794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,1,0.007526399940252304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4,128,0.006326399743556976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,128,0.009644799679517747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4,64,0.005075199902057648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,64,0.009014400094747544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4,32,0.004527999833226204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,32,0.008604799956083297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4,16,0.0042975999414920805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,16,0.008291199803352356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4,8,0.0040895998477935794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,8,0.007980799674987793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4,4,0.00411520004272461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,4,0.00759039968252182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4,2,0.003961599990725517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,2,0.00785600021481514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4,1,0.0040224000811576845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4,1,0.0076223999261856076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8,128,0.00623680017888546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,128,0.010329599678516387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8,64,0.005056000128388405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,64,0.009312000125646591
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8,32,0.004502400010824204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,32,0.008752000331878663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8,16,0.004176000133156776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,16,0.008048000186681748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8,8,0.00398080013692379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,8,0.007791999727487564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8,4,0.00398080013692379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,4,0.0076895996928215025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8,2,0.003936000168323517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,2,0.007875200361013412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8,1,0.004038399830460548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8,1,0.007814399898052216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,16,128,0.006252799928188324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2,4,0.004262400045990944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,16,64,0.005062400177121162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,128,0.011667200177907944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,16,32,0.004476799815893173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,32,0.008873599767684936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,16,16,0.004358400031924248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,16,0.008531200140714646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,16,8,0.004147199913859367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,8,0.00835840031504631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,16,4,0.004086399823427201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,4,0.00806720033288002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,16,2,0.00398080013692379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,2,0.007718399912118912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,16,1,0.004044799879193306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,1,0.007651200145483017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,32,128,0.006496000289916992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,128,0.015324799716472626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,32,64,0.005369599908590317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,64,0.01167680025100708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,32,32,0.004649600014090538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,32,0.009600000083446502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,32,16,0.0044064000248909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,16,0.009385599941015243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,32,8,0.004054399952292442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,8,0.008131200075149536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,32,4,0.004016000032424927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,4,0.00806720033288002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,32,2,0.004070400074124336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,2,0.00788159966468811
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,32,1,0.0039680000394582745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,32,1,0.008064000308513642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,48,128,0.007353600114583969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,128,0.019180800020694732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,48,64,0.005750399827957153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,64,0.013683199882507324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,48,32,0.004886399954557419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,32,0.01019200012087822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,48,16,0.004476799815893173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,32,0.007833600044250488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,16,0.008928000181913375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,8,0.00838399976491928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,48,4,0.004080000147223472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,8,0.007398399710655213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,4,0.008220800012350083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,48,2,0.00411520004272461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,2,0.007913599908351897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,48,1,0.0040640000253915785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,48,1,0.00783040001988411
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,64,128,0.007859200239181519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,4,0.00729919970035553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,128,0.02271360009908676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,2,0.007731200009584427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,64,0.015408000349998474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,64,32,0.005955199897289276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,32,0.011427199840545655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,64,16,0.006332799792289734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,16,0.009443199634552002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,64,8,0.0058975998312234875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,64,4,0.006275200098752975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,48,8,0.00605119988322258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,4,0.008316799998283386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,64,2,0.005939200147986412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,2,0.008287999778985977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,64,1,0.0058432001620531086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,1,0.0081216000020504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,80,128,0.008697599917650223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,80,64,0.006431999802589417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,128,0.02640959918498993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,80,32,0.006611199676990509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,64,0.01709440052509308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,64,0.008294399827718735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,32,0.012755200266838074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,16,0.00928959995508194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,80,8,0.004412800073623657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,80,4,0.004316800087690353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,8,0.008105599880218506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2,16,0.007782399654388428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,4,0.00857279971241951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,80,1,0.0050976000726222995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,2,0.00793280005455017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,80,1,0.008073599636554718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,128,0.02963840067386627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,96,64,0.0067391999065876005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,64,0.018665599822998046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,96,32,0.005257600173354149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,32,0.013020800054073333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,96,16,0.004704000055789947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,16,64,0.009958399832248688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,16,0.010361599922180175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,64,64,0.005980800092220307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,96,8,0.004428799822926521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,8,0.009116800129413604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,96,4,0.00432640016078949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,96,2,0.004335999861359597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,96,1,0.004163200035691261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,2,0.008163200318813324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,1,0.00825600028038025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,128,128,0.011228799819946289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,128,0.037190398573875426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,128,64,0.007737600058317184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,64,8,0.008630400151014328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,64,0.022364799678325654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,128,32,0.005715199932456017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,128,16,0.004729599878191948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,32,0.014860799908638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,128,8,0.004371200129389763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,16,0.010819199681282043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,8,0.009043200314044953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,128,4,0.004265600070357323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,4,0.008371199667453765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,128,2,0.004134399816393852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,2,0.008191999793052674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,128,1,0.004239999875426293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,128,1,0.007993599772453308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,96,128,0.00910080000758171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,160,128,0.015296000242233276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,128,0.046697598695755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,160,64,0.008326400071382523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,160,32,0.006086400151252747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,64,0.026159998774528504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,160,16,0.004816000163555145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,32,0.017110399901866913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,16,0.012303999811410903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,160,4,0.004416000097990036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,8,0.009888000041246413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,4,0.00894080027937889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,160,2,0.004467200115323066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,2,0.008540800213813782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,160,1,0.004284799844026565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,160,1,0.008348800241947174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,80,16,0.006137600168585777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,192,128,0.020937600731849672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,192,32,0.006521599739789963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,192,64,0.008908800035715102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,64,0.029552000761032104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,192,16,0.005203200131654739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,32,0.01873600035905838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,192,8,0.004681599885225296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,16,0.012761600315570831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,8,0.010089600086212158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,192,4,0.004361600056290627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,192,2,0.004220800101757049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,2,0.008291199803352356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,192,1,0.004198399931192398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,1,0.008265600353479386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,256,128,0.026368001103401185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,256,64,0.009971199929714203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,128,0.07145280241966248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,64,0.03669120073318481
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,256,32,0.007280000299215317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,32,0.02213120013475418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,256,16,0.005715199932456017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,16,0.015139199793338776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,256,8,0.004755200073122978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,256,4,0.004444799944758415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,8,0.011231999844312668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,4,0.00926399976015091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,256,2,0.004307200014591217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,256,1,0.0041600000113248825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,2,0.008604799956083297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,96,4,0.008262400329113007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,256,1,0.008489599823951722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,320,128,0.029462400078773498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,320,64,0.012620800733566284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,128,0.08643199801445008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,320,32,0.007587199658155441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,64,0.0466048002243042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,320,16,0.0060479998588562015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,32,0.025494399666786193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,16,0.01659200042486191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,320,8,0.0049183998256921765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,8,0.012083200365304947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,320,4,0.004419200122356415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,80,2,0.0039391998201608654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,320,2,0.004262400045990944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,2,0.008713600039482117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,320,1,0.004086399823427201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,1,0.008534400165081025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,384,128,0.033913600444793704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,384,64,0.016016000509262086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,128,0.10255680084228516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,64,0.055961602926254274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,384,32,0.008857599645853042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,32,0.029289600253105164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,384,16,0.006486400216817856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,384,8,0.004870399832725525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,16,0.018726399540901183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,8,0.013260799646377563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,384,4,0.0044064000248909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,160,8,0.0051807999610900875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,4,0.010230399668216705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,384,2,0.004454400017857551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,2,0.008844800293445587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,512,128,0.04254400134086609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,384,1,0.0052928000688552855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,384,1,0.008470399677753449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,512,64,0.022870400547981264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,512,32,0.010038399696350097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,128,0.13276480436325072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,512,16,0.007168000191450119
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,32,0.03696640133857727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,128,0.058361601829528806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,512,8,0.005446400120854377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,512,4,0.0047807998955249785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,8,0.01491519957780838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,512,2,0.004416000097990036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,2,0.009068799763917923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,4,0.01088000014424324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,512,1,0.004371200129389763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,1,0.008377599716186523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,768,128,0.057251197099685666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,768,64,0.031411200761795044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,192,4,0.00880960002541542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,768,32,0.014521600306034088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,64,0.10225919485092164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,768,16,0.00854720026254654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,128,0.19593919515609742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,16,0.029516801238059998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,8,0.018559999763965607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,32,0.05581120252609253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,768,2,0.00565119981765747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,4,0.013120000064373017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,768,1,0.0062015999108552934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,2,0.010294400155544281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,768,1,0.008937600255012512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1024,128,0.0735967993736267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1024,64,0.03958080112934113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,128,0.25892798900604247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,64,0.1324128031730652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1024,32,0.02168319970369339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1024,16,0.010252799838781357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,32,0.07055040001869202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1024,8,0.007142399996519088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,8,0.02186879962682724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1024,4,0.00533440001308918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,4,0.014703999459743499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1024,2,0.004582399874925614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,2,0.010784000158309937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1024,1,0.0043136000633239744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,1,0.009276799857616425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1536,128,0.10416640043258667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,128,0.38461759090423586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,64,0.07097600102424621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,512,16,0.02223999947309494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1536,64,0.05587520003318787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1536,32,0.030444800853729248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,64,0.19430079460144042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1536,16,0.014521600306034088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,16,0.05527359843254089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1536,8,0.008720000088214875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,8,0.029241600632667543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1536,4,0.006441599875688553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,4,0.01820800006389618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,768,8,0.006255999952554703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1536,2,0.005296000093221664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,2,0.012854400277137756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,768,4,0.005215999856591225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,1536,1,0.004553600028157234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,1,0.010355199873447418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2048,64,0.07060800194740295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2048,128,0.13430720567703247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2048,32,0.038380798697471616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,64,0.2575167894363403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,32,0.1312608003616333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,16,0.07071359753608704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2048,8,0.009939199686050415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2048,4,0.007238399982452392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,8,0.03703039884567261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,4,0.022147199511528014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2048,2,0.0057599999010562895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,2,0.014892800152301789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,1,0.010972800105810166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,3072,128,0.19541759490966798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1024,16,0.037222400307655334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,3072,64,0.10185920000076294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,320,4,0.009724800288677216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,64,0.3837152004241943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,32,0.1947711944580078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,3072,16,0.030031999945640563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,16,0.10223040580749512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,3072,8,0.01438400000333786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,8,0.05485759973526001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,3072,4,0.008908800035715102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,4,0.0298880010843277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,3072,2,0.006563200056552887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,2,0.018662400543689728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,3072,1,0.006003199890255928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,1536,32,0.10218240022659301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,1,0.013078400492668152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4096,128,0.25938560962677004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4096,64,0.13309439420700073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2048,16,0.02117439955472946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,128,1.0332703590393066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4096,32,0.0692255973815918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,2048,128,0.5104415893554688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4096,16,0.03798080086708069
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,64,0.5086815834045411
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4096,8,0.02006399929523468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,16,0.1314847946166992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4096,4,0.009996800124645234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,8,0.07047680020332336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,4,0.0368800014257431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,2048,1,0.005286400020122528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4096,1,0.005571199953556061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,2,0.02200320065021515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,1,0.014927999675273895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,6144,128,0.38327679634094236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,6144,64,0.19448959827423096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,6144,32,0.10060160160064698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,32,0.3818176031112671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,3072,32,0.05395519733428955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,64,0.7735167980194092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,6144,16,0.053711998462677005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,128,1.548089599609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,6144,8,0.02942720055580139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,6144,4,0.014316800236701965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,8,0.10152959823608398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,16,0.1944416046142578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,6144,1,0.006284800171852112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,2,0.02957119941711426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,1,0.018457600474357606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8192,128,0.5079616069793701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8192,64,0.2690047979354858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8192,32,0.1312831997871399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,64,1.0234432220458984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,128,2.0930912017822267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8192,16,0.06906239986419678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,32,0.5082687854766845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8192,8,0.03804480135440826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8192,4,0.019571200013160706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,16,0.25702080726623533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,8,0.13178240060806273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8192,2,0.010124800354242324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,4,0.07075520157814026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,2,0.036595198512077334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,8192,1,0.007283200323581695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,8192,1,0.02234880030155182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,4096,32,0.25658559799194336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,4096,2,0.0073472000658512115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,3072,128,0.772166395187378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,fp8,6144,4,0.05510720014572144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_pre,default,bfloat16,6144,2,0.008739200234413148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1,128,0.006291200220584869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1,64,0.005958399921655655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,64,0.008185599744319916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,32,0.008326400071382523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1,16,0.004569600149989128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,16,0.00782719999551773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1,8,0.004383999854326248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,8,0.007875200361013412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1,4,0.004550400003790855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,4,0.007936000078916549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1,2,0.004521600157022476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,2,0.008060800284147263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1,1,0.004569600149989128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,1,0.008131200075149536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2,128,0.006003199890255928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,128,0.00942080020904541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2,64,0.0052767999470233916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,64,0.00868159979581833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2,32,0.004870399832725525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1,128,0.008848000317811966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,32,0.00830719992518425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,16,0.00801599994301796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2,8,0.0044319998472929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,8,0.00809279978275299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2,4,0.004620800167322159
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,4,0.008275199681520462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2,2,0.004527999833226204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,1,0.008022399991750718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,128,0.009667199850082398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4,32,0.004825599864125252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,32,0.008495999872684479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4,16,0.004473600164055824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,16,0.00825600028038025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4,8,0.0043680001050233844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,8,0.007993599772453308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4,4,0.004566400125622749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2,16,0.004969599843025208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,4,0.008284799754619598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4,2,0.005071999877691269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,2,0.008275199681520462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4,1,0.0051807999610900875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,1,0.008111999928951263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8,128,0.006089600175619126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,128,0.010147199779748917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8,64,0.005036799982190132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8,32,0.0050016000866889955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,64,0.00897279977798462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8,16,0.004476799815893173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1,32,0.004879999905824661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,16,0.009043200314044953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,32,0.00854720026254654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8,8,0.0043935999274253845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8,4,0.004982399940490723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8,2,0.005033599957823753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,2,0.008073599636554718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8,1,0.005116799846291542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,1,0.008035200089216233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16,128,0.006188800185918808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,128,0.011846400052309036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16,64,0.0051231998950243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,64,0.009836799651384353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16,32,0.006076800078153611
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,32,0.009097599983215332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2,1,0.0046431999653577805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16,16,0.005046400055289268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16,8,0.005030399933457374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4,64,0.0050592001527547835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,8,0.008336000144481659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,4,0.008287999778985977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16,2,0.00490880012512207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,2,0.00793280005455017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16,1,0.005033599957823753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,1,0.00817599967122078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,32,128,0.006662400066852569
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,128,0.015337599813938141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,32,64,0.00535999983549118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,8,0.008163200318813324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,64,0.011264000087976456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,32,32,0.004662400111556053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,32,0.009510400146245957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,32,16,0.004572800174355507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,16,0.008787199854850769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,32,8,0.004521600157022476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,32,4,0.00456320010125637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,8,0.008399999886751174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,32,2,0.005065599828958512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,4,0.008560000360012055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,2,0.008271999657154083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,32,1,0.0044895999133586885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,32,1,0.008156800270080566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,48,128,0.006911999732255936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,128,0.018646399676799773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,48,64,0.00535999983549118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,64,0.013382400572299957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,48,32,0.0046847999095916745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,32,0.01043199971318245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,48,16,0.0047807998955249785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,48,8,0.004611200094223023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,16,0.009279999881982803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,48,4,0.005030399933457374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,8,0.008646400272846222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,4,0.008646400272846222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,48,2,0.004540799930691719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,2,0.008511999994516373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,48,1,0.004572800174355507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,48,1,0.008320000022649765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,64,128,0.007283200323581695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,128,0.02184640020132065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,64,64,0.005561599880456925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,64,0.01454080045223236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,64,32,0.0048096001148223875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4,128,0.005875200033187866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,32,0.010892800241708755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,64,16,0.0046431999653577805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,16,0.009305600076913834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16,4,0.004403200000524521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,8,0.009120000153779983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,4,0.008684799820184708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,64,2,0.0045471999794244765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,2,0.00854720026254654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,64,1,0.004582399874925614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,64,1,0.008291199803352356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,80,128,0.007711999863386154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,128,0.02510400116443634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,80,64,0.005756799876689911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,64,0.016249600052833556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,80,32,0.004975999891757965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,32,0.012121599912643433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,80,16,0.004854400083422661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,16,0.010035199671983719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,80,8,0.004761600121855736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,8,0.009232000261545182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,80,4,0.0046720001846551895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,4,0.008691199868917466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,80,2,0.004438399896025657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,2,0.008182399719953538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,80,1,0.004476799815893173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,80,1,0.008195199817419053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,96,128,0.008054400235414505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,128,0.028518399596214293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,64,8,0.004550400003790855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,96,64,0.006054399907588959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,96,32,0.005011200159788132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,64,4,0.004412800073623657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,32,0.013091200590133667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,96,16,0.004889599978923798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,16,0.010428799688816071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,96,8,0.004598399996757508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,8,0.009279999881982803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,96,4,0.004620800167322159
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,96,2,0.004560000076889992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8,4,0.007731200009584427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,96,1,0.005046400055289268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2,2,0.007699199765920639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,1,0.008406399935483932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,128,0.0380160003900528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,128,64,0.00639680027961731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,64,0.021055999398231506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,128,32,0.0051392000168561935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,32,0.014451199769973755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,128,16,0.0050016000866889955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,16,0.011244799941778183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,128,8,0.004844800010323525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,8,0.009609600156545639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,128,4,0.005100800096988678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,4,0.00904960036277771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,128,2,0.004524800181388855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,2,0.008454400300979614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,128,1,0.0046271998435258865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,128,1,0.008428800106048583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,160,128,0.017574399709701538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,128,128,0.009244800359010697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,128,0.046054399013519286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,160,64,0.0069311998784542085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16,16,0.008233600109815598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,64,0.024710400402545928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,160,32,0.005763199925422668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,160,16,0.005135999992489815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,32,0.01570879966020584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,16,0.01218239963054657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,8,0.010092800110578537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,160,4,0.0046016000211238865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,160,2,0.004678399860858917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,4,0.009040000289678574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,160,1,0.0046847999095916745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,2,0.00836160033941269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,160,1,0.008211199939250947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,192,128,0.02152000069618225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,192,64,0.0074432000517845156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,128,0.05394880175590515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,64,0.0282368004322052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,32,0.017679999768733978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,192,16,0.005209600180387497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,16,0.013033600151538849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,4,0.008588799834251403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,8,0.010524799674749374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,192,4,0.004543999955058098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,192,2,0.004460800066590309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4,64,0.008339200168848038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,4,0.00936639979481697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,2,0.008537600189447403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,192,1,0.008217599987983704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,256,128,0.025190401077270507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,128,0.0695743978023529
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,256,64,0.007987199723720551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,64,0.03632639944553375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,256,32,0.006012799963355064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,32,0.021180799603462218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,256,16,0.005046400055289268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,16,0.014591999351978302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,256,8,0.0047775998711586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,8,0.010812799632549285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,256,4,0.004656000062823296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,4,0.009337600320577621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,256,2,0.004560000076889992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,2,0.008726400136947633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,256,1,0.004470400139689445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,256,1,0.008761599659919739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,320,128,0.029260799288749695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,128,0.08573439717292786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,320,64,0.009132800251245498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,64,0.044924798607826236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,320,32,0.006972800195217133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,160,8,0.00493439994752407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,320,16,0.005398400127887726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,16,0.016047999262809753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,320,8,0.005238400027155876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,192,32,0.005744000151753426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,8,0.012185599654912949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,320,4,0.00514880008995533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,4,0.00966079980134964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,192,8,0.0050592001527547835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,320,2,0.005129599943757057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,320,1,0.004524800181388855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,1,0.008828800171613693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,384,128,0.03301759958267212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,384,64,0.013279999792575835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,128,0.10310080051422119
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,384,32,0.0071263998746871945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,64,0.05251839756965637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,384,16,0.00544000007212162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,16,0.017660799622535705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,384,8,0.0054848000407218935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,8,0.013062399625778199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,384,4,0.004982399940490723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,4,0.01043199971318245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,384,2,0.004656000062823296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,2,0.009308800101280212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,384,1,0.0046431999653577805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,1,0.008806400001049042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,512,128,0.04102720022201538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,128,0.13385599851608276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,512,64,0.023014399409294128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,64,0.06885120272636414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,512,32,0.008051200211048127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,32,0.035071998834609985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,192,1,0.0044319998472929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,512,16,0.005984000116586685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,512,8,0.005100800096988678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,16,0.02041600048542023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,8,0.014710399508476257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,512,4,0.004720000177621841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,4,0.011155200004577637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,32,0.02463040053844452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,512,2,0.004572800174355507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,2,0.00926399976015091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,64,0.018156799674034118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,512,1,0.00862400010228157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,768,128,0.0566976010799408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,768,64,0.03017599880695343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,64,0.10094399452209472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,768,32,0.009779199957847595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,128,0.1959328055381775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,32,0.05261440277099609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,96,2,0.008566399663686752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,768,8,0.005449600145220756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,768,16,0.007152000069618225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,16,0.027952000498771667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,8,0.017590400576591492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,4,0.012732799351215362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,768,2,0.004764800146222115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,320,2,0.009036800265312195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,2,0.010127999633550645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,768,1,0.009097599983215332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1024,128,0.07250880002975464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1024,64,0.03809280097484589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1024,32,0.020979200303554536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,128,0.2577728033065796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,64,0.13230719566345214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1024,16,0.008115199953317642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,32,0.06838399767875672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,384,32,0.027779200673103334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1024,8,0.005929600074887275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,8,0.021091200411319733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1024,4,0.00586559996008873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1024,2,0.005052800104022026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,4,0.01470080018043518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1024,1,0.004771199822425842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,2,0.01130559965968132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,1,0.00958079993724823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1536,64,0.054790401458740236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1536,128,0.10242240428924561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,128,0.38617279529571535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1536,32,0.029203200340270997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,64,0.19687360525131226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1536,16,0.009852799773216247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1536,8,0.007407999783754349
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,16,0.05249279737472534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,32,0.10003199577331542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,8,0.028143998980522156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,4,0.017654399573802947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1536,4,0.00554560013115406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1536,1,0.004960000142455101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,2,0.012889599800109864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1536,1,0.010278400033712387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2048,128,0.13223999738693237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2048,64,0.06935999989509582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,128,0.5233664035797119
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,64,0.2575135946273804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2048,32,0.03686720132827759
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2048,16,0.018361599743366243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,32,0.13115839958190917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,16,0.06786879897117615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2048,8,0.00820159986615181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2048,4,0.005859199911355972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,4,0.02115200012922287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2048,2,0.005078399926424027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,2,0.014595200121402741
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,2048,1,0.004844800010323525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,512,1,0.004681599885225296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,1,0.010982400178909302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,3072,128,0.19269440174102784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,3072,64,0.09818559885025024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,768,4,0.005177599936723709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,128,0.8138367652893066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,3072,32,0.05321599841117859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,768,1,0.004441599920392036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,3072,16,0.02841919958591461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,3072,8,0.009833600372076035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,64,0.3848864078521729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,32,0.1945248007774353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,8,0.05232959985733032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,16,0.10059839487075806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,3072,4,0.007238399982452392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,3072,2,0.005465599894523621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,3072,1,0.0049632001668214795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,4,0.027910399436950683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,2,0.017849600315093993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,3072,1,0.01295360028743744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,1024,16,0.03517760038375854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4096,128,0.2602751970291138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4096,64,0.12818559408187866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4096,32,0.06748800277709961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,64,0.5232255935668946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,32,0.25645439624786376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4096,8,0.017552000284194947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,16,0.13083200454711913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4096,4,0.008128000050783157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4096,2,0.005766399949789047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,4,0.035235199332237246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,128,1.1136672019958496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,2,0.020921599864959717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4096,1,0.005091200023889542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,1,0.014716799557209014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,6144,64,0.18634560108184814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,1536,2,0.005209600180387497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,6144,32,0.09775360226631165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,64,0.8186335563659668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,32,0.3824448108673096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,6144,16,0.058582401275634764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,16,0.1950976014137268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,128,1.629542350769043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,6144,8,0.027644801139831542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,8,0.09942079782485962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,4,0.0521888017654419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,6144,4,0.009907200187444686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,6144,2,0.007360000163316727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,2,0.02775999903678894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,2048,8,0.03519040048122406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,6144,1,0.005395200103521347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,6144,1,0.017798399925231932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8192,64,0.24412479400634765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8192,32,0.1261952042579651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,64,1.108694362640381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,32,0.5218751907348633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8192,16,0.0676576018333435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8192,8,0.0359360009431839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,16,0.2548192024230957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8192,4,0.01703680008649826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,8,0.13065919876098633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,4,0.06761279702186584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8192,2,0.008191999793052674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,2,0.035062399506568906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8192,1,0.006025600060820579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,1,0.021279999613761903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,12288,64,0.35987200736999514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,4096,16,0.03623999953269959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,12288,128,0.7291552066802979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,4096,8,0.06785280108451844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,12288,32,0.18417919874191285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,12288,16,0.09686080217361451
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,16,0.38568639755249023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,32,0.8200480461120605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,12288,8,0.05194240212440491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,64,1.6967872619628905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,12288,4,0.02791999876499176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,8,0.19811199903488158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,4,0.10371520519256591
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,2,0.052451199293136595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,12288,1,0.007388799637556076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,1,0.028038400411605834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,12288,128,3.379635238647461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16384,64,0.553875207901001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16384,128,0.9684543609619141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,8192,128,0.5283360004425048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16384,32,0.28320960998535155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16384,16,0.14522240161895753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,32,1.1053055763244628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,16,0.5250080108642579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16384,8,0.0667680025100708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,8192,128,2.180860710144043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16384,4,0.03618879914283753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16384,2,0.018089599907398224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,8,0.2563647985458374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,16384,1,0.007987199723720551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,64,2.173583984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,4,0.1341312050819397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,1,0.03617599904537201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,128,4.380419158935547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,20480,64,0.5936287879943848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,20480,32,0.3026304006576538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,6144,128,0.37109758853912356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,20480,128,1.3322367668151855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,20480,16,0.1545151948928833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,12288,2,0.009737599641084671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,20480,8,0.08254079818725586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,20480,4,0.04338879883289337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,8,0.3198911905288696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,16,0.6698847770690918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,20480,2,0.02336000055074692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,bfloat16,20480,1,0.008739200234413148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,4,0.16375360488891602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,32,1.344320011138916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,1,0.043161600828170776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,2,0.0864799976348877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,64,2.8257471084594727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,20480,128,5.467715072631836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_gen_post,default,fp8,16384,2,0.06778240203857422
