framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,1,0.012708799540996551
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,1,0.012055999785661697
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,1,0.012097600102424621
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,1,0.01210239976644516
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,1,0.012185599654912949
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,1,0.012080000340938568
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,1,0.01852159947156906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,1,0.017635199427604675
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,3,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,3,0.01207199990749359
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,1,0.011910399794578553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,3,0.01202400028705597
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,3,0.012137600034475327
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,3,0.012135999649763108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,3,0.018436799943447112
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,3,0.01783200055360794
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,3,0.017556799948215483
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,3,0.017577600479125977
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,3,0.01764799952507019
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,3,0.01765599995851517
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,3,0.01757279932498932
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,7,0.012880000472068786
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,7,0.012353599816560746
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,7,0.012127999961376191
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,7,0.011896000057458878
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,7,0.011910399794578553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,7,0.011952000111341477
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,7,0.011902400106191636
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,7,0.01830720007419586
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,7,0.018113599717617036
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,7,0.017811200022697447
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,7,0.01751520037651062
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,7,0.017348800599575043
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,7,0.017558400332927705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,7,0.017470400035381316
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,15,0.01292639970779419
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,15,0.01239679977297783
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,15,0.012257599830627441
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,15,0.012166400253772736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,15,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,15,0.012188799679279327
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,15,0.01199359968304634
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,15,0.01815039962530136
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,15,0.017820799350738527
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,15,0.017796799540519714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,15,0.017900800704956053
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,15,0.01754080057144165
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,15,0.01767839938402176
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,15,0.017798399925231932
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,31,0.012676799297332763
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,31,0.012249600142240524
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,31,0.012120000272989272
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,31,0.01199520006775856
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,31,0.012203200161457062
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,31,0.012150400131940842
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,31,0.012172800302505494
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,31,0.01844000071287155
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,31,0.017739200592041017
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,31,0.017609600722789765
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,31,0.017880000174045563
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,31,0.017785599827766417
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,31,0.017657600343227386
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,31,0.017560000717639922
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,63,0.012774400413036346
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,63,0.012252800166606903
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,63,0.012388800084590913
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,63,0.012247999757528305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,63,0.012219200283288956
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,63,0.01223199963569641
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,63,0.012196800112724305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,63,0.018484799563884734
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,63,0.0177839994430542
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,63,0.017748799920082093
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,63,0.018166400492191315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,63,0.017769600450992584
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,63,0.01794559955596924
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,63,0.017905600368976593
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,127,0.014560000598430633
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,127,0.014075200259685516
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,127,0.013900800049304963
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,127,0.013887999951839447
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,127,0.01377120018005371
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,127,0.013976000249385834
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,127,0.014043200016021728
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,127,0.02025440037250519
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,127,0.019735999405384064
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,127,0.01964640021324158
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,127,0.019420799612998963
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,127,0.019358399510383605
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,127,0.019393600523471832
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,127,0.019739200174808503
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,255,0.017793600261211396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,3,0.011984000355005265
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,255,0.01709440052509308
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,255,0.017126399278640746
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,255,0.01706240028142929
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,255,0.017083199322223665
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,255,0.017046399414539337
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,255,0.023291200399398804
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,255,0.02284799963235855
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,1,0.017584000527858735
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,1,0.017796799540519714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,255,0.022759999334812164
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,255,0.022729599475860597
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,1,0.017446400225162507
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,1,0.017419199645519256
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,255,0.02276960015296936
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,1,0.0176704004406929
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,511,0.017748799920082093
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,511,0.016728000342845918
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,511,0.0176704004406929
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,511,0.016896000504493712
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,511,0.017793600261211396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,511,0.01780640035867691
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,511,0.02476480007171631
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,511,0.023889599740505217
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,511,0.023095999658107758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,511,0.02248159945011139
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,511,0.023798400163650514
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,511,0.0238319993019104
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,511,0.0239439994096756
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,1023,0.01920959949493408
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,1023,0.017950400710105896
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,1023,0.01700959950685501
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,1023,0.01775840073823929
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,1023,0.017833599448204042
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,1023,0.017843200266361235
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,1023,0.025481599569320678
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,1023,0.02449920028448105
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,1023,0.023668800294399262
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,1023,0.02306559979915619
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,1023,0.024057599902153014
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,1023,0.02420639991760254
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,1023,0.02417600005865097
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,3,0.012190400063991547
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,2047,0.01993599981069565
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,255,0.022657600045204163
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,255,0.02287199944257736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,2047,0.018614399433135986
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,2047,0.017556799948215483
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,2047,0.01815200001001358
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,511,0.018828800320625304
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,2047,0.017953599989414214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,2047,0.028379198908805848
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,2047,0.02627519965171814
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,2047,0.02582719922065735
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,2047,0.026419198513031004
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,255,0.017246399819850922
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,2047,0.02650400102138519
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,4095,0.020553599298000335
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,4095,0.0196383997797966
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,4095,0.022017599642276765
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,4095,0.01964160054922104
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,4095,0.019843199849128725
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,4095,0.020494399964809416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,4095,0.020059199631214143
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,4095,0.033083200454711914
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,1023,0.017180800437927246
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,4095,0.031241598725318908
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,4095,0.030771198868751525
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,4095,0.031244799494743347
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,4095,0.03162240087985992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,4095,0.031281599402427675
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,8191,0.02491360008716583
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,8191,0.023472000658512116
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,8191,0.02263360023498535
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,8191,0.021803200244903564
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,8191,0.023345600068569183
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,8191,0.024153600633144378
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,8191,0.0420879989862442
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,8191,0.03993119895458221
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,8191,0.02364159971475601
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,8191,0.03949599862098694
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,8191,0.0386927992105484
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,8191,0.040617600083351135
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,8191,0.041043201088905336
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,16383,0.02971999943256378
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,16383,0.02621600031852722
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,16383,0.025366398692131042
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,16383,0.025760000944137572
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,16383,0.02545279860496521
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,2047,0.01797440052032471
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,16383,0.02550080120563507
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,16383,0.06285920143127441
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,16383,0.06000639796257019
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,2047,0.01820639967918396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,16383,0.054980802536010745
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,16383,0.05493280291557312
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,16383,0.05474079847335815
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,16383,0.055435198545455935
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,32767,0.03702400028705597
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,32767,0.03246079981327057
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,2047,0.0265855997800827
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,32767,0.031195199489593504
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,32767,0.030428799986839294
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,32767,0.030665600299835206
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,32767,0.030681601166725157
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,32767,0.0996832013130188
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,32767,0.09620000123977661
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,32767,0.09582560062408448
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,4095,0.031006398797035217
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,32767,0.09515039920806885
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,32767,0.09522079825401306
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,32767,0.09513599872589111
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,65535,0.0481904000043869
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,65535,0.045212799310684205
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,8191,0.04129279851913452
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,65535,0.04429279863834381
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,65535,0.04376479983329773
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,65535,0.04284160137176514
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,16383,0.025303998589515687
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,65535,0.043131199479103086
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,65535,0.042403200268745424
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,65535,0.16625759601593018
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,65535,0.16333760023117067
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,65535,0.1631440043449402
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,65535,0.16257920265197753
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,16383,0.055144000053405764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,2047,0.0272271990776062
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,131071,0.06668800115585327
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,131071,0.06336960196495056
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,131071,0.06025599837303162
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,131071,0.060577601194381714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,131071,0.06042240262031555
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,131071,0.0618287980556488
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,131071,0.060361599922180174
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,32767,0.09546719789505005
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,131071,0.2966687917709351
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,131071,0.2938256025314331
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,131071,0.29333760738372805
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,131071,0.2926687955856323
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,131071,0.29224319458007814
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,1,0.012636800110340119
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,1,0.01226240023970604
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,131071,0.2917392015457153
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,1,0.012036799639463424
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,1,0.011912000179290772
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,1,0.012094400078058242
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,1,0.012145599722862244
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,1,0.018331199884414673
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,1,0.017766399681568144
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,1,0.017558400332927705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,1,0.017267200350761413
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,1,0.01706559956073761
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,1,0.01720480024814606
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,1,0.017382399737834932
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,3,0.012831999361515046
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,65535,0.16268479824066162
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,65535,0.16243200302124022
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,3,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,3,0.012051200121641159
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,3,0.012116800248622894
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,3,0.012097600102424621
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,3,0.012135999649763108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,3,0.018084800243377684
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,3,0.017718400061130523
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,3,0.0176144003868103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,3,0.017422400414943695
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,3,0.017643199861049653
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,3,0.01727840006351471
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,3,0.01736319959163666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,7,0.012880000472068786
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,7,0.012337599694728852
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,7,0.012174399942159653
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,7,0.012070400267839431
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,7,0.012147200107574464
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,7,0.012185599654912949
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,7,0.018299199640750885
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,7,0.017479999363422392
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,7,0.01761920005083084
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,7,0.01741439998149872
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,7,0.017399999499320983
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,7,0.0174575999379158
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,7,0.01740639954805374
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,15,0.012971200048923492
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,15,0.012636800110340119
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,15,0.012164799869060517
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,15,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,15,0.012318400293588638
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,15,0.012316799908876418
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,15,0.018566399812698364
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,15,0.017735999822616578
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,65535,0.1642848014831543
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,15,0.017467199265956877
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,15,0.017820799350738527
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,15,0.017555199563503265
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,3,0.012428800016641617
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,31,0.012945599853992462
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,31,0.012390399724245072
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,31,0.012140800058841706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,31,0.012095999717712403
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,31,0.011984000355005265
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,31,0.012039999663829803
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,31,0.012200000137090683
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,31,0.018329599499702455
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,31,0.017871999740600587
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,31,0.017360000312328337
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,31,0.017214399576187134
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,31,0.017420800030231477
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,31,0.017566399276256563
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,131071,0.291646409034729
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,31,0.017318400740623473
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,63,0.01286720037460327
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,1,0.011860799789428712
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,63,0.012353599816560746
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,15,0.012139199674129486
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,63,0.011988800019025803
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,63,0.012160000205039979
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,63,0.011851199716329575
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,63,0.012163200229406358
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,63,0.018185600638389587
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,63,0.01781439930200577
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,63,0.01728159934282303
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,63,0.017795200645923614
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,63,0.01723040044307709
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,63,0.017740799486637114
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,63,0.01725279986858368
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,127,0.01462399959564209
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,127,0.013996799290180207
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,127,0.013915200531482697
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,127,0.01385439932346344
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,127,0.013900800049304963
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,127,0.01366879940032959
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,127,0.013899199664592743
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,127,0.01988479942083359
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,127,0.019683200120925903
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,127,0.018972800672054292
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,127,0.01937440037727356
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,127,0.018955199420452117
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,127,0.01931679993867874
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,127,0.019105599820613862
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,32767,0.03033599853515625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,255,0.016979199647903443
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,255,0.01703519970178604
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,255,0.016991999745368958
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,255,0.016916799545288085
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,255,0.017059199512004852
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,255,0.023291200399398804
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,255,0.02266560047864914
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,255,0.02247840017080307
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,255,0.0226623997092247
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,255,0.022524799406528472
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,255,0.0226160004734993
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,255,0.022492800652980805
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,511,0.01873600035905838
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,511,0.017825600504875184
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,63,0.012028799951076507
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,511,0.017078399658203125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,511,0.017852799594402315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,511,0.018062399327754976
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,511,0.017884799838066102
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,15,0.01785759925842285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,511,0.02513279914855957
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,511,0.024091200530529024
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,511,0.023244799673557283
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,511,0.022918400168418885
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,511,0.024439999461174013
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,15,0.01763039976358414
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,1023,0.019524799287319185
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,1023,0.018240000307559966
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,1023,0.017543999850749968
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,1023,0.017236800491809846
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,255,0.017664000391960144
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,1023,0.017923200130462648
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,1023,0.02791680097579956
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,1023,0.02656799852848053
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,1023,0.025526401400566102
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,7,0.01218400001525879
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,1023,0.025140801072120668
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,1023,0.025940799713134767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,1023,0.026049599051475525
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,2047,0.02041279971599579
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,2047,0.018995200097560883
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,2047,0.018118399381637573
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,2047,0.017875200510025023
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,2047,0.018062399327754976
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,2047,0.018308800458908082
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,2047,0.018478399515151976
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,511,0.016755199432373045
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,2047,0.030211201310157774
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,2047,0.02926880121231079
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,2047,0.02900640070438385
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,2047,0.02913439869880676
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,511,0.024052800238132478
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,2047,0.029393601417541503
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,2047,0.02961280047893524
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,4095,0.023043200373649597
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,511,0.02420479953289032
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,4095,0.020798400044441223
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,4095,0.020260800421237946
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,4095,0.0194255992770195
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,4095,0.02067680060863495
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,1023,0.017867200076580048
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,1023,0.017953599989414214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,4095,0.03807680010795593
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,255,0.01671359986066818
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,4095,0.03680799901485443
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,4095,0.03765600025653839
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,4095,0.03760960102081299
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,8191,0.0264272004365921
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,1023,0.026212799549102783
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,8191,0.023419199883937834
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,8191,0.022801600396633148
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,8191,0.022166399657726286
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,8191,0.024905599653720856
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,8191,0.025516799092292784
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,8191,0.025668799877166748
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,8191,0.059982401132583615
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,8191,0.053774398565292356
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,8191,0.05213919878005981
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,8191,0.05137119889259338
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,8191,0.054020798206329344
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,8191,0.054718399047851564
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,8191,0.05464159846305847
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,16383,0.033062401413917544
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,16383,0.02865920066833496
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,4095,0.039636799693107606
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,4095,0.036959999799728395
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,4095,0.03747200071811676
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,16383,0.030454400181770324
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,16383,0.029585599899291992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,16383,0.09583200216293335
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,16383,0.09285600185394287
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,16383,0.09195039868354797
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,16383,0.09155200123786926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,16383,0.0937712013721466
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,16383,0.09441919922828675
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,2047,0.031779199838638306
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,16383,0.09424480199813842
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,32767,0.04467360079288483
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,32767,0.0414463996887207
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,32767,0.04010399878025055
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,32767,0.03810240030288696
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,32767,0.04069760143756866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,32767,0.041073599457740785
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,32767,0.040487998723983766
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,32767,0.16272640228271484
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,32767,0.15930880308151246
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,32767,0.1587280035018921
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,4095,0.020136000216007234
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,4095,0.020366400480270386
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,32767,0.1610319972038269
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,16383,0.027710399031639098
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,32767,0.16027040481567384
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,65535,0.06267520189285278
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,16383,0.02698560059070587
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,65535,0.05766720175743103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,65535,0.05611199736595154
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,65535,0.057361602783203125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,16383,0.029044800996780397
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,65535,0.05822399854660034
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,65535,0.29241600036621096
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,65535,0.2899791955947876
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,65535,0.2894079923629761
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,65535,0.2901760101318359
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,65535,0.29100799560546875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,65535,0.29078240394592286
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,131071,0.09931359887123108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,131071,0.09341599941253662
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,131071,0.09223999977111816
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,131071,0.09095360040664673
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,131071,0.09239199757575989
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,131071,0.09301120042800903
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,131071,0.09323359727859497
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,131071,0.5502863883972168
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,131071,0.5481071949005127
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,131071,0.545636796951294
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,32767,0.15796960592269899
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,131071,0.5444128036499023
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,32767,0.1598415970802307
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,131071,0.5473696231842041
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,1,0.012751999497413635
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,1,0.012356799840927125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,1,0.011999999731779098
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,1,0.012142399698495865
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,131071,0.5467040061950683
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,1,0.012140800058841706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,65535,0.05797280073165893
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,1,0.017708800733089447
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,1,0.017608000338077544
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,1,0.01781280040740967
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,65535,0.2898400068283081
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,1,0.017980800569057466
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,1,0.017888000607490538
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,3,0.012556800246238708
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,3,0.012368000298738479
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,3,0.012140800058841706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,3,0.012247999757528305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,3,0.01228799968957901
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,3,0.012252800166606903
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,3,0.018571199476718904
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,3,0.018251200020313264
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,3,0.01793919950723648
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,3,0.017859199643135072
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,3,0.017979200184345245
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,3,0.017953599989414214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,3,0.017888000607490538
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,7,0.013107199966907502
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,7,0.012590399384498597
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,7,0.012387199699878693
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,131071,0.5470352172851562
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,7,0.012120000272989272
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,1,0.012451200187206269
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,7,0.012137600034475327
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,65535,0.058798402547836304
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,7,0.012164799869060517
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,7,0.018513600528240203
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,1,0.018068799376487733
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,7,0.017828799784183502
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,7,0.017534400522708892
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,7,0.01767839938402176
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,7,0.017825600504875184
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,7,0.017903999984264375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,15,0.013046400249004364
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,15,0.012643200159072877
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,15,0.012220799922943115
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,15,0.012308800220489502
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,15,0.012065599858760833
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,3,0.013072000443935394
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,15,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,15,0.012039999663829803
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,15,0.018489600718021394
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,15,0.01823520064353943
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,15,0.017910400032997133
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,15,0.017790399491786957
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,15,0.017923200130462648
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,15,0.01794400066137314
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,15,0.017686399817466735
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,31,0.013150399923324585
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,31,0.012228800356388092
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,31,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,31,0.012113600224256515
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,31,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,31,0.012086399644613267
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,31,0.018753600120544434
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,31,0.01823520064353943
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,31,0.017852799594402315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,31,0.018004800379276275
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,31,0.017902399599552154
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,31,0.01791680008172989
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,31,0.01791519969701767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,63,0.012563200294971466
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,63,0.012406399846076966
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,63,0.012319999933242797
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,63,0.01241919994354248
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,63,0.012337599694728852
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,63,0.012398400157690049
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,63,0.01890240013599396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,63,0.01826079934835434
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,1,0.012012799829244613
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,7,0.012398400157690049
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,63,0.018276800215244294
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,1,0.01855199933052063
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,63,0.01799200028181076
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,63,0.01802079975605011
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,127,0.014403200149536133
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,127,0.014192000031471252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,127,0.013923199474811554
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,127,0.013870400190353394
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,127,0.013955199718475341
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,127,0.013896000385284425
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,127,0.020470400154590607
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,31,0.012518399953842163
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,127,0.020172800123691558
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,127,0.019945600628852846
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,127,0.01961120069026947
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,127,0.01961279958486557
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,127,0.019755199551582336
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,255,0.01791680008172989
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,255,0.01720480024814606
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,255,0.01706240028142929
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,255,0.017015999555587767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,255,0.017113600671291352
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,255,0.017140799760818483
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,255,0.017084799706935883
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,255,0.023873600363731384
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,255,0.023255999386310577
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,255,0.02300959974527359
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,63,0.018083199858665466
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,255,0.023105600476264955
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,63,0.017604799568653108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,255,0.023212799429893495
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,7,0.01820639967918396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,511,0.019358399510383605
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,511,0.01801439970731735
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,511,0.01732639968395233
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,127,0.014972800016403198
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,511,0.01690720021724701
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,511,0.017927999794483184
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,511,0.018464000523090364
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,511,0.02783840000629425
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,511,0.026977598667144775
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,511,0.02579360008239746
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,511,0.025118398666381835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,511,0.02619520127773285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,511,0.026633599400520326
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,511,0.026161599159240722
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,1023,0.019871999323368073
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,1023,0.01777919977903366
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,1023,0.017265599966049195
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,1023,0.018243199586868285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,1023,0.018459199368953703
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,1023,0.018177600204944612
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,1023,0.0310479998588562
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,1023,0.029787200689315795
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,63,0.012990400195121765
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,1023,0.029100799560546876
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,1023,0.02839359939098358
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,1023,0.029382398724555968
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,1023,0.02942880094051361
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,2047,0.0215488001704216
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,2047,0.019934399425983428
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,2047,0.019020800292491914
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,2047,0.018587200343608855
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,2047,0.01899999976158142
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,2047,0.018691200017929076
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,2047,0.01886879950761795
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,255,0.02319519966840744
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,255,0.02312159985303879
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,2047,0.03718400001525879
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,2047,0.036366400122642514
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,2047,0.03586559891700745
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,2047,0.03599520027637482
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,2047,0.036078399419784545
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,4095,0.02454880028963089
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,4095,0.022068800032138826
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,1023,0.018240000307559966
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,4095,0.02099040001630783
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,4095,0.022473600506782532
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,127,0.01977279931306839
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,4095,0.022392000257968902
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,4095,0.022332799434661866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,4095,0.05883359909057617
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,4095,0.05301920175552368
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,4095,0.05062239766120911
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,4095,0.050256001949310306
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,4095,0.05146399736404419
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,1023,0.02943840026855469
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,8191,0.03328480124473572
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,8191,0.027577599883079527
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,8191,0.025377601385116577
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,8191,0.025441598892211915
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,8191,0.028911998867988585
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,8191,0.02917119860649109
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,8191,0.02915999889373779
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,8191,0.09485759735107421
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,511,0.018217599391937254
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,8191,0.09168480038642883
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,8191,0.09028319716453552
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,8191,0.09480640292167664
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,4095,0.021371200680732727
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,8191,0.09477760195732117
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,16383,0.043673598766326906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,16383,0.03964479863643646
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,16383,0.03826560080051422
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,16383,0.03677760064601898
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,16383,0.04071840047836304
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,16383,0.04084320068359375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,16383,0.04095839858055115
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,16383,0.16097279787063598
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,4095,0.05114560127258301
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,16383,0.15702400207519532
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,16383,0.1559712052345276
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,4095,0.051665598154067995
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,16383,0.15918879508972167
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,16383,0.160315203666687
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,16383,0.16045440435409547
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,32767,0.06203200221061707
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,32767,0.0577888011932373
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,32767,0.055990397930145264
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,32767,0.05456799864768982
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,32767,0.05774719715118408
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,32767,0.05845280289649964
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,32767,0.058169597387313844
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,2047,0.03864479959011078
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,32767,0.29167680740356444
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,32767,0.2884191989898682
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,32767,0.287062406539917
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,32767,0.2906431913375854
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,32767,0.29103360176086424
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,8191,0.09013760089874268
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,65535,0.09101759791374206
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,32767,0.29119679927825926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,8191,0.09356319904327393
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,65535,0.09060159921646119
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,65535,0.0879967987537384
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,65535,0.09036319851875305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,65535,0.09268640279769898
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,65535,0.5473135948181153
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,65535,0.5452383995056153
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,65535,0.5433184146881104
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,16383,0.15723520517349243
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,65535,0.5428976058959961
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,65535,0.5460896015167236
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,65535,0.5468592166900634
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,65535,0.5477663993835449
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,131071,0.16375199556350709
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,131071,0.16151039600372313
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,131071,0.1569599986076355
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,131071,0.15668799877166747
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,131071,0.15753599405288696
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,131071,0.15827200412750245
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,131071,0.15828160047531128
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,32767,0.28775200843811033
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,2047,0.03612160086631775
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,65535,0.09539039731025696
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,131071,1.057551956176758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,131071,1.056492805480957
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,131071,1.0520992279052734
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,65535,0.0926751971244812
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,1,0.013121600449085235
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,131071,1.0509407997131348
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,1,0.012484800070524216
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,1,0.012088000029325485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,1,0.012291199713945388
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,131071,1.0549344062805175
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,1,0.018963199853897095
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,1,0.018249599635601042
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,1,0.018003199994564057
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,1,0.018062399327754976
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,131071,1.0556768417358398
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,1,0.018219199776649476
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,3,0.012831999361515046
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,1,0.017990399897098542
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,3,0.012465599924325943
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,3,0.012230399996042252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,3,0.012296000123023986
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,3,0.01223360002040863
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,3,0.01228479966521263
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,3,0.01231200024485588
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,3,0.01892479956150055
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,3,0.018534399569034576
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,3,0.018248000741004945
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,3,0.018131199479103088
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,3,0.018268799781799315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,3,0.018248000741004945
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,7,0.01289760023355484
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,7,0.012455999851226807
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,7,0.012263999879360199
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,7,0.012140800058841706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,7,0.012097600102424621
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,7,0.012256000190973282
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,7,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,7,0.01886560022830963
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,7,0.01833280026912689
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,7,0.018203200399875642
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,7,0.018030400574207305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,1,0.012825599312782288
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,7,0.017852799594402315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,1,0.012350399792194367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,7,0.017822399735450745
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,1,0.012328000366687774
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,15,0.013020800054073333
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,15,0.012723200023174286
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,15,0.012415999919176102
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,15,0.012374400347471236
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,15,0.012193600088357926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,15,0.012268800288438797
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,15,0.018911999464035035
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,1,0.018449600040912627
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,15,0.01857919991016388
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,15,0.01820160001516342
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,15,0.01809599995613098
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,15,0.01812160015106201
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,15,0.018287999927997588
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,31,0.01305440068244934
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,31,0.012800000607967377
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,31,0.01247519999742508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,31,0.012510399520397186
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,31,0.012435200065374375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,31,0.012332800030708312
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,31,0.012375999987125397
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,31,0.018830400705337525
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,31,0.018323199450969697
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,31,0.018302400410175324
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,3,0.017820799350738527
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,31,0.018177600204944612
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,31,0.018057599663734436
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,31,0.018308800458908082
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,63,0.01319040060043335
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,63,0.012993599474430084
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,63,0.012532800436019897
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,63,0.012585599720478059
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,63,0.012388800084590913
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,63,0.012601600587368011
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,63,0.012478400021791458
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,63,0.019299200177192687
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,63,0.018943999707698823
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,63,0.018408000469207764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,63,0.018352000415325163
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,63,0.018510399758815764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,63,0.018356800079345703
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,127,0.014934399724006652
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,127,0.014313599467277527
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,127,0.014326399564743042
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,127,0.014259199798107147
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,127,0.014281600713729858
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,127,0.014071999490261078
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,127,0.02094399929046631
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,7,0.0179967999458313
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,127,0.0206496000289917
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,127,0.020491200685501098
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,15,0.012318400293588638
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,127,0.02043360024690628
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,127,0.02030239999294281
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,131071,1.055361557006836
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,127,0.020310400426387785
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,15,0.018278400599956512
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,255,0.017433600127696992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,255,0.01733279973268509
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,255,0.01709440052509308
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,255,0.017161600291728973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,255,0.017076799273490907
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,255,0.017166399955749513
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,255,0.026020801067352294
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,255,0.025572800636291505
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,31,0.0186256006360054
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,255,0.025414401292800905
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,255,0.0254287987947464
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,255,0.025260800123214723
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,511,0.019926400482654573
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,511,0.01844799965620041
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,511,0.017824000120162962
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,511,0.017235200107097625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,511,0.018331199884414673
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,63,0.01854719966650009
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,511,0.018644799292087556
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,511,0.0316431999206543
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,127,0.014009599387645722
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,511,0.030084800720214844
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,511,0.028992000222206115
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,511,0.028747200965881348
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,511,0.02969760000705719
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,511,0.030112001299858093
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,511,0.03003999888896942
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,1023,0.021503999829292297
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,1023,0.019441600143909454
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,1023,0.018863999843597413
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,1023,0.0180976003408432
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,1023,0.01913760006427765
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,1023,0.019276799261569978
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,1023,0.018731200695037843
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,1023,0.039447999000549315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,127,0.020265600085258482
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,1023,0.03623999953269959
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,1023,0.03552959859371185
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,1023,0.03628000020980835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,1023,0.03644160032272339
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,1023,0.03630239963531494
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,2047,0.025180798768997193
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,2047,0.020768000185489653
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,2047,0.019918400049209594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,2047,0.020483200252056123
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,2047,0.020585599541664123
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,2047,0.02093600034713745
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,2047,0.059064000844955444
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,2047,0.05336160063743591
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,2047,0.05059199929237366
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,2047,0.04996159970760346
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,2047,0.050139200687408444
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,2047,0.05011039972305298
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,511,0.01860480010509491
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,4095,0.02643359899520874
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,4095,0.025284799933433532
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,4095,0.024604800343513488
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,4095,0.026009601354599
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,4095,0.026495999097824095
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,4095,0.026982399821281432
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,4095,0.0944927990436554
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,4095,0.09129279851913452
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,1023,0.0368800014257431
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,4095,0.0900048017501831
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,255,0.018004800379276275
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,4095,0.0913424015045166
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,4095,0.09126240015029907
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,4095,0.0919055998325348
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,2047,0.02154559940099716
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,8191,0.043639999628067014
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,8191,0.0389519989490509
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,8191,0.036062398552894594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,8191,0.04064480066299438
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,8191,0.040785598754882815
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,255,0.025248000025749208
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,8191,0.04141600131988525
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,8191,0.16068799495697023
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,8191,0.15718560218811034
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,8191,0.15549440383911134
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,8191,0.15962719917297363
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,8191,0.16046719551086425
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,8191,0.16098560094833375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,16383,0.0612496018409729
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,4095,0.03284800052642822
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,16383,0.056732797622680665
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,16383,0.05487520098686218
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,16383,0.054179197549819945
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,16383,0.05766080021858215
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,16383,0.05845119953155518
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,16383,0.05808479785919189
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,16383,0.2918031930923462
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,16383,0.28670558929443357
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,16383,0.286680006980896
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,16383,0.29016640186309817
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,4095,0.08938559889793396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,16383,0.2911983966827393
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,16383,0.2915760040283203
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,8191,0.03710399866104126
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,32767,0.0960864007472992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,32767,0.09123039841651917
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,32767,0.08964639902114868
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,32767,0.08903200030326844
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,32767,0.0936352014541626
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,32767,0.09281759858131408
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,8191,0.15640480518341066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,32767,0.09337120056152344
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,32767,0.549886417388916
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,32767,0.5450128078460693
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,32767,0.5434976100921631
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,32767,0.5468639850616455
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,32767,0.5466832160949707
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,16383,0.2879663944244385
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,65535,0.16382399797439576
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,65535,0.1544368028640747
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,65535,0.15363839864730836
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,65535,0.15736639499664307
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,65535,0.15813599824905394
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,255,0.025460800528526305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,2047,0.05065600275993347
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,65535,1.057412815093994
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,65535,1.0541664123535157
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,65535,1.0514703750610352
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,32767,0.5430880069732666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,65535,1.051467227935791
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,131071,0.2993056058883667
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,32767,0.5477215766906738
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,65535,1.0551679611206055
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,65535,1.0563424110412598
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,65535,0.15763360261917114
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,131071,0.2907151937484741
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,131071,0.29309279918670655
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,131071,0.28993279933929444
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,131071,0.28975520133972166
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,65535,0.15786720514297486
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,131071,2.075713539123535
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,131071,2.0719535827636717
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,131071,2.07006721496582
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,131071,2.0696880340576174
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,1,0.013180799782276154
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,1,0.012511999905109405
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,1,0.012771199643611907
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,1,0.012275200337171555
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,1,0.012545600533485413
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,1,0.012352000176906585
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,1,0.012596799433231354
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,1,0.01900160014629364
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,65535,1.0547439575195312
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,1,0.018240000307559966
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,1,0.01812160015106201
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,131071,0.29191200733184813
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,1,0.018118399381637573
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,3,0.013195200264453888
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,1,0.018031999468803406
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,3,0.012734399735927581
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,3,0.01252799928188324
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,3,0.012332800030708312
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,3,0.012508800625801087
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,3,0.012375999987125397
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,3,0.012495999783277511
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,3,0.018947200477123262
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,3,0.018372799456119537
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,3,0.018329599499702455
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,3,0.017977599799633027
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,131071,2.0743648529052736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,3,0.018003199994564057
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,131071,2.07543830871582
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,7,0.013395200669765472
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,3,0.01797119975090027
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,7,0.012689599394798278
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,7,0.012824000418186187
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,7,0.012307199835777282
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,7,0.01239520013332367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,7,0.01903519928455353
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,7,0.018614399433135986
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,7,0.018287999927997588
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,7,0.018089599907398224
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,7,0.018063999712467194
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,7,0.01826079934835434
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,7,0.018328000605106354
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,15,0.013326400518417358
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,131071,2.0717344284057617
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,15,0.012774400413036346
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,15,0.012457600235939026
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,15,0.012468799948692322
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,15,0.012593600153923034
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,15,0.012468799948692322
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,15,0.01897760033607483
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,1,0.01851679980754852
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,15,0.01873279958963394
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,15,0.018219199776649476
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,15,0.01796640008687973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,1,0.017876799404621124
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,15,0.01791680008172989
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,15,0.017995199561119078
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,15,0.017961600422859193
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,31,0.01404000073671341
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,31,0.012803199887275695
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,31,0.012720000743865967
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,31,0.012444800138473511
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,31,0.01257600039243698
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,31,0.012707200646400452
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,31,0.012563200294971466
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,31,0.019809600710868836
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,31,0.018753600120544434
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,31,0.01809599995613098
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,31,0.01823520064353943
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,31,0.018374399840831758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,31,0.018302400410175324
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,3,0.017927999794483184
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,63,0.0133775994181633
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,63,0.012705600261688233
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,63,0.012747199833393097
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,63,0.01263359934091568
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,63,0.012804800271987915
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,63,0.012494400143623352
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,7,0.012547199428081513
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,7,0.01242400035262108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,63,0.01953279972076416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,63,0.019284799695014954
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,63,0.018694399297237395
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,63,0.01884640008211136
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,63,0.018774400651454925
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,63,0.019092799723148347
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,127,0.014790399372577668
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,127,0.014763200283050537
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,127,0.014476799964904785
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,127,0.01451839953660965
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,127,0.01419840008020401
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,127,0.014190399646759033
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,127,0.022977599501609804
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,15,0.012603199481964112
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,127,0.02266719937324524
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,127,0.022492800652980805
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,127,0.022385600209236144
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,127,0.0221903994679451
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,127,0.022300800681114195
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,127,0.022275200486183165
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,255,0.01900320053100586
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,255,0.017795200645923614
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,255,0.01762399971485138
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,255,0.01759999990463257
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,255,0.017710399627685548
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,255,0.01765599995851517
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,255,0.017539200186729432
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,255,0.02895039916038513
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,255,0.02826719880104065
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,255,0.02813439965248108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,255,0.02831520140171051
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,31,0.018251200020313264
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,255,0.028092798590660096
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,255,0.02813439965248108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,511,0.022040000557899474
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,511,0.01920959949493408
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,511,0.0184688001871109
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,63,0.012595200538635254
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,511,0.018134400248527527
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,511,0.019331200420856474
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,511,0.019340799748897554
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,511,0.01934240013360977
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,511,0.039032000303268435
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,511,0.03628639876842499
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,511,0.03487359881401062
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,511,0.035873600840568544
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,511,0.03620319962501526
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,511,0.03640480041503906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,1023,0.02534559965133667
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,1023,0.021247999370098115
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,1023,0.020662400126457214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,1023,0.01983039975166321
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,1023,0.02112639993429184
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,1023,0.021118399500846863
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,1023,0.059996801614761355
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,1023,0.05272960066795349
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,1023,0.04986880123615265
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,1023,0.04854080080986023
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,131071,0.28997440338134767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,1023,0.049911999702453615
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,2047,0.032867199182510375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,1023,0.05016639828681946
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,2047,0.02431199997663498
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,2047,0.02369759976863861
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,255,0.028489598631858827
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,2047,0.024792000651359558
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,2047,0.024809600412845613
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,2047,0.09526079893112183
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,2047,0.09077759981155395
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,2047,0.08949120044708252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,63,0.01950239986181259
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,2047,0.08887680172920227
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,511,0.035780799388885495
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,2047,0.08959519863128662
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,2047,0.08954560160636901
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,2047,0.08998240232467651
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,127,0.015340800583362579
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,4095,0.038764798641204835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,4095,0.03678399920463562
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,4095,0.03522560000419617
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,4095,0.03760479986667633
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,4095,0.038089600205421445
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,4095,0.038185599446296695
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,4095,0.16099679470062256
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,4095,0.1563040018081665
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,4095,0.15512479543685914
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,4095,0.15424319505691528
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,1023,0.0500544011592865
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,4095,0.15642080307006836
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,4095,0.15669440031051635
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,4095,0.156987202167511
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,2047,0.026284798979759216
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,8191,0.05396639704704285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,8191,0.05260800123214722
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,8191,0.05735359787940979
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,8191,0.05836319923400879
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,2047,0.024691200256347655
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,8191,0.291592001914978
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,8191,0.28777759075164794
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,8191,0.28614559173583987
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,8191,0.2861263990402222
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,8191,0.28981919288635255
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,4095,0.04481280148029328
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,8191,0.29149279594421384
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,16383,0.09670720100402833
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,16383,0.09134719967842102
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,1023,0.02083359956741333
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,16383,0.08806719779968261
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,16383,0.08683680295944214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,16383,0.09234399795532226
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,16383,0.0935584008693695
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,16383,0.09300640225410461
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,16383,0.5469279766082764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,16383,0.5438591957092285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,16383,0.5434927940368652
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,8191,0.06280959844589233
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,8191,0.05656480193138123
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,32767,0.16399840116500855
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,16383,0.546395206451416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,16383,0.5487247943878174
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,16383,0.546996784210205
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,32767,0.15697599649429322
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,32767,0.1562991976737976
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,32767,0.15732799768447875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,32767,0.1592527985572815
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,32767,0.1587056040763855
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,8191,0.29055840969085694
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,32767,1.0555328369140624
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,32767,1.052342414855957
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,32767,1.0506272315979004
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,65535,0.29641759395599365
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,32767,1.0517727851867675
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,32767,1.0547648429870606
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,32767,1.0549872398376465
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,32767,1.0567791938781739
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,65535,0.29035999774932864
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,16383,0.5408927917480468
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,8191,0.05804799795150757
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,65535,0.28147039413452146
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,65535,0.28842079639434814
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,65535,0.28935201168060304
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,32767,0.1528607964515686
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,65535,2.070955276489258
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,65535,2.0665615081787108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,65535,2.0652944564819338
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,65535,2.0639312744140623
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,65535,2.069438362121582
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,65535,2.0726463317871096
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,65535,0.2845072031021118
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,131071,0.5580863952636719
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,131071,0.5489312171936035
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,131071,0.5493296146392822
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,65535,0.29007198810577395
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,131071,0.5541920185089111
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,131071,0.5510816097259521
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,65535,2.071651268005371
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,131071,0.5503583908081054
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,131071,0.5514800071716308
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,1,0.015039999783039094
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,1,0.013767999410629273
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,1,0.013094399869441987
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,1,0.012943999469280243
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,1,0.012940800189971924
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,131071,4.1057086944580075
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,131071,4.100236892700195
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,131071,4.097462463378906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,1,0.020283199846744537
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,1,0.019182400405406953
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,1,0.018849599361419677
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,1,0.018932799994945525
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,1,0.018620799481868743
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,131071,4.121700668334961
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,1,0.018934400379657747
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,131071,4.101313781738281
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,131071,4.100920104980469
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,3,0.013529600203037262
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,3,0.014579200744628906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,3,0.013211199641227722
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,3,0.013289600610733032
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,3,0.01321599930524826
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,3,0.013116799294948578
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,3,0.01923999935388565
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,3,0.018811200559139252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,3,0.01903360038995743
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,3,0.019012799859046935
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,3,0.01896799951791763
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,3,0.018955199420452117
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,7,0.01446239948272705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,7,0.013575999438762665
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,7,0.01292639970779419
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,7,0.012891200184822083
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,7,0.01287200003862381
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,7,0.012928000092506409
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,7,0.012908799946308136
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,7,0.02077440023422241
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,7,0.019222399592399596
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,7,0.018932799994945525
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,7,0.01875839978456497
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,7,0.018849599361419677
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,7,0.01894560009241104
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,7,0.019032000005245207
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,15,0.014791999757289887
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,15,0.013473600149154663
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,15,0.013444800674915314
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,15,0.013334399461746216
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,15,0.013016000390052795
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,15,0.013470399379730224
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,15,0.013315199315547943
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,15,0.02102559953927994
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,15,0.019572800397872923
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,15,0.019201600551605226
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,15,0.01897120028734207
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,15,0.01926880031824112
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,15,0.01918559968471527
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,15,0.019276799261569978
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,31,0.014688000082969666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,31,0.013847999274730682
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,31,0.013451200723648072
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,31,0.013052800297737121
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,31,0.013176000118255616
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,31,0.013502399623394012
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,31,0.013092799484729767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,1,0.012907199561595917
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,31,0.019755199551582336
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,131071,4.096995162963867
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,31,0.019388799369335175
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,31,0.01977279931306839
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,31,0.019228799641132353
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,31,0.019337600469589232
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,3,0.013075199723243714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,63,0.014182400703430176
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,63,0.015083199739456177
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,63,0.013910399377346038
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,63,0.013670399785041809
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,63,0.013787199556827546
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,63,0.013792000710964203
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,63,0.023004800081253052
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,63,0.021963199973106383
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,63,0.02168319970369339
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,63,0.02157599925994873
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,63,0.021483199298381807
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,63,0.021704000234603883
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,63,0.021615999937057494
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,127,0.016883200407028197
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,127,0.015851199626922607
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,127,0.015563200414180755
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,127,0.015326400101184846
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,127,0.015380799770355225
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,127,0.015374399721622467
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,127,0.027347201108932497
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,127,0.0264847993850708
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,127,0.02603360116481781
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,31,0.021031999588012697
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,127,0.026259198784828186
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,1,0.013038399815559387
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,127,0.026196798682212828
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,255,0.020252799987792967
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,255,0.018852800130844116
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,255,0.01897599995136261
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,255,0.01849119961261749
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,31,0.019412800669670105
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,255,0.018748800456523895
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,255,0.018595199286937713
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,255,0.03654879927635193
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,255,0.03540000021457672
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,3,0.020310400426387785
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,255,0.035051199793815616
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,63,0.013484799861907959
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,255,0.035062399506568906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,255,0.034908801317214966
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,255,0.03508000075817108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,511,0.027532801032066345
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,511,0.022598400712013245
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,511,0.021007999777793884
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,511,0.021139200031757354
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,127,0.025990399718284606
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,511,0.021649600565433504
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,511,0.021247999370098115
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,127,0.025942400097846985
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,511,0.06214560270309448
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,511,0.05426880121231079
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,511,0.0503600001335144
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,255,0.018403199315071107
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,511,0.050220799446105954
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,511,0.0506991982460022
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,511,0.050419199466705325
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,1023,0.035543999075889586
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,1023,0.023790399730205535
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,1023,0.023129600286483764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,255,0.03504480123519897
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,1023,0.02370399981737137
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,1023,0.023715199530124666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,511,0.020351999998092653
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,1023,0.09726240038871765
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,127,0.015457600355148315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,1023,0.09076799750328064
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,1023,0.08717439770698547
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,1023,0.08870880007743835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,1023,0.08865119814872742
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,1023,0.08871039748191833
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,2047,0.04699999988079071
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,2047,0.03911679983139038
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,2047,0.03702239990234375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,2047,0.035703998804092404
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,2047,0.03638240098953247
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,2047,0.03712959885597229
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,2047,0.16364480257034303
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,511,0.05173599720001221
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,2047,0.15573760271072387
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,1,0.01866080015897751
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,2047,0.15403679609298707
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,2047,0.15529600381851197
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,2047,0.15575360059738158
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,1023,0.026531198620796205
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,4095,0.06482719779014587
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,4095,0.057678401470184326
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,4095,0.05453280210494995
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,4095,0.05308960080146789
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,4095,0.055099201202392575
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,4095,0.05564000010490418
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,4095,0.05500159859657287
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,4095,0.2938240051269531
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,4095,0.28849599361419676
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,4095,0.28690080642700194
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,1023,0.08912320137023926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,4095,0.28544480800628663
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,4095,0.2869839906692505
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,4095,0.2880880117416382
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,8191,0.09915840029716491
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,4095,0.2879631996154785
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,8191,0.08684160113334656
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,8191,0.09262400269508361
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,8191,0.09321600198745728
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,2047,0.03662559986114502
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,8191,0.09301919937133789
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,2047,0.15770399570465088
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,8191,0.550603199005127
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,8191,0.5449552059173584
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,2047,0.15512000322341918
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,8191,0.5415311813354492
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,8191,0.5467743873596191
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,1023,0.023665599524974823
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,16383,0.1673087954521179
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,16383,0.15856000185012817
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,8191,0.5469183921813965
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,16383,0.15607520341873168
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,16383,0.15354559421539307
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,16383,0.16051199436187744
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,16383,0.1600592017173767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,8191,0.09170399904251099
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,16383,1.0586544036865235
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,16383,1.0531935691833496
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,16383,1.0508591651916503
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,16383,1.050540828704834
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,8191,0.5428143978118897
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,16383,1.0547183990478515
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,32767,0.289083194732666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,16383,1.0559823989868165
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,8191,0.5474512100219726
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,16383,1.0553919792175293
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,32767,0.2841615915298462
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,32767,0.2837615966796875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,32767,0.28845279216766356
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,32767,0.29109759330749513
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,8191,0.08851680159568787
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,32767,2.074398422241211
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,32767,2.0694175720214845
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,32767,2.0650352478027343
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,32767,2.0643856048583986
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,32767,0.2962399959564209
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,32767,2.0685855865478517
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,32767,0.2901936054229736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,16383,0.1584239959716797
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,65535,0.5476399898529053
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,65535,0.547815990447998
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,65535,0.5501743793487549
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,32767,2.070705604553223
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,65535,0.5502016067504882
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,65535,0.5625328063964844
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,65535,0.55698561668396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,32767,2.069443130493164
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,65535,0.5508399963378906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,65535,4.10559196472168
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,65535,4.0981182098388675
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,65535,4.0987903594970705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,1,0.015332800149917603
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,1,0.014257599413394929
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,1,0.013780799508094788
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,1,0.013923199474811554
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,1,0.014022399485111237
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,1,0.014105600118637086
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,1,0.020094400644302367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,1,0.01960480064153671
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,1,0.019652800261974336
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,1,0.019513599574565887
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,1,0.019486400485038757
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,3,0.015105600655078887
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,3,0.014159999787807465
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,3,0.013967999815940857
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,3,0.01401119977235794
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,3,0.014192000031471252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,3,0.013876800239086152
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,3,0.014020800590515137
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,3,0.021086399257183076
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,3,0.019788800179958342
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,3,0.01980479955673218
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,3,0.01964640021324158
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,3,0.020262399315834047
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,3,0.019828799366950988
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,65535,4.101236724853516
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,65535,4.102366256713867
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,7,0.015399999916553497
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,7,0.014347200095653535
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,1,0.014043200016021728
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,7,0.014134399592876434
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,7,0.014095999300479889
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,7,0.014097599685192109
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,7,0.013846400380134582
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,7,0.021331200003623964
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,7,0.01984799951314926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,1,0.019678400456905366
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,7,0.020137600600719452
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,7,0.01972000002861023
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,7,0.019675199687480927
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,7,0.01944800019264221
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,15,0.015385599434375763
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,15,0.014271999895572662
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,15,0.013864000141620637
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,15,0.014297600090503692
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,15,0.014004799723625182
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,15,0.014051200449466705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,65535,4.111617660522461
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,15,0.021772800385951994
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,15,0.020175999402999877
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,65535,4.1013023376464846
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,15,0.020769600570201874
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,15,0.020742399990558623
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,15,0.02048799991607666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,15,0.020470400154590607
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,31,0.014507199823856353
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,31,0.015756799280643462
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,3,0.01980320066213608
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,31,0.014156800508499146
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,31,0.013857600092887879
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,31,0.014020800590515137
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,31,0.014132800698280334
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,31,0.023470400273799895
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,7,0.014060799777507783
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,31,0.022270399332046508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,1,0.021140800416469575
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,31,0.022256000339984892
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,7,0.02000479996204376
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,31,0.022200000286102296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,63,0.016579200327396394
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,63,0.014763200283050537
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,63,0.01467359960079193
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,63,0.014286400377750396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,63,0.014327999949455262
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,15,0.014286400377750396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,63,0.02680160105228424
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,63,0.02560639977455139
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,63,0.02547520101070404
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,63,0.025174400210380553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,63,0.025241601467132568
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,63,0.025326400995254517
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,63,0.02558720111846924
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,127,0.018267199397087097
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,127,0.01653759926557541
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,127,0.016337600350379945
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,127,0.01627040058374405
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,127,0.016539199650287627
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,31,0.01419679969549179
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,127,0.016033600270748138
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,127,0.016126400232315062
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,127,0.03460640013217926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,127,0.03306719958782196
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,31,0.022302399575710296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,127,0.03325439989566803
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,31,0.022294400632381438
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,127,0.03308480083942413
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,127,0.0331167995929718
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,127,0.032995200157165526
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,255,0.02123199999332428
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,255,0.019633600115776063
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,255,0.019543999433517457
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,255,0.01966080069541931
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,63,0.01449279934167862
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,255,0.019950400292873382
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,255,0.019592000544071196
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,255,0.019382399320602418
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,255,0.05260159969329834
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,255,0.048214399814605714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,63,0.014737600088119506
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,255,0.048511999845504764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,255,0.04778079986572266
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,255,0.047788798809051514
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,255,0.04859359860420227
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,511,0.03954240083694458
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,511,0.03065119981765747
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,511,0.023393599689006804
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,511,0.02544800043106079
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,511,0.025415998697280884
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,511,0.026211199164390565
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,511,0.10079519748687744
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,15,0.020260800421237946
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,511,0.0900592029094696
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,511,0.08882880210876465
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,511,0.08994240164756775
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,511,0.09073920249938965
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,511,0.09077600240707398
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,1023,0.05026559829711914
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,1023,0.04204800128936768
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,1023,0.03834240138530731
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,1023,0.036399999260902406
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,1023,0.03761279881000519
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,1023,0.03841600120067597
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,1023,0.038673600554466246
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,127,0.03282720148563385
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,1023,0.1596511960029602
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,1023,0.1567247986793518
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,1023,0.15527839660644532
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,1023,0.15650399923324584
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,1023,0.15740159749984742
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,1023,0.15764479637145995
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,255,0.04909279942512512
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,2047,0.06049280166625977
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,2047,0.0562720000743866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,2047,0.05461599826812744
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,2047,0.05496799945831299
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,511,0.025355198979377748
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,2047,0.05591040253639221
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,511,0.09340639710426331
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,2047,0.29908480644226076
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,2047,0.2911776065826416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,2047,0.287337589263916
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,2047,0.2870879888534546
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,2047,0.28751521110534667
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,2047,0.28790080547332764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,4095,0.10527039766311645
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,4095,0.09038720130920411
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,4095,0.08783360123634339
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,4095,0.09081599712371827
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,1023,0.16658400297164916
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,4095,0.09389280080795288
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,4095,0.09275360107421875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,31,0.02223999947309494
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,4095,0.5486432075500488
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,4095,0.5440639972686767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,2047,0.06865919828414917
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,4095,0.5422944068908692
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,4095,0.5458096027374267
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,4095,0.5470272064208984
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,8191,0.17215679883956908
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,8191,0.1629520058631897
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,8191,0.15435999631881714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,8191,0.16153440475463868
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,8191,0.16506240367889405
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,8191,0.16516159772872924
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,2047,0.28803040981292727
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,4095,0.0949999988079071
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,8191,1.064475154876709
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,8191,1.0556672096252442
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,8191,1.0533679962158202
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,4095,0.5536831855773926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,8191,1.051633644104004
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,2047,0.05518239736557007
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,8191,1.057356834411621
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,4095,0.5466368198394775
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,8191,1.0607968330383302
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,16383,0.29208319187164306
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,8191,1.0605664253234863
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,16383,0.28903200626373293
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,16383,0.29092159271240237
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,16383,0.29450559616088867
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,16383,2.0794559478759767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,16383,2.0709999084472654
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,16383,2.0678688049316407
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,16383,2.067790412902832
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,16383,0.30053761005401614
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,8191,0.15747679471969606
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,16383,0.28432478904724123
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,16383,0.29636640548706056
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,32767,0.5651631832122803
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,32767,0.5525279998779297
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,32767,0.5514272212982178
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,16383,2.0752399444580076
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,32767,0.5488944053649902
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,32767,0.551854419708252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,32767,0.5552000045776367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,32767,0.5554800033569336
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,16383,2.070790481567383
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,16383,2.077403259277344
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,1,0.017979200184345245
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,32767,4.109827041625977
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,32767,4.102531051635742
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,1,0.015508800745010376
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,32767,4.098158264160157
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,1,0.015556800365447997
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,32767,4.096347045898438
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,1,0.0159168004989624
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,1,0.015676799416542053
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,32767,4.105998229980469
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,1,0.0234047994017601
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,1,0.021694399416446686
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,1,0.021211199462413788
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,1,0.020975999534130096
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,3,0.017475199699401856
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,3,0.0161423996090889
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,3,0.01573439985513687
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,3,0.015408000349998474
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,3,0.01555200070142746
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,3,0.015852800011634825
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,3,0.015404799580574035
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,3,0.023163199424743652
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,3,0.0218639999628067
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,3,0.021971200406551362
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,3,0.02163040041923523
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,3,0.021691200137138367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,3,0.021583999693393707
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,3,0.021854400634765625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,7,0.017532800137996674
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,32767,4.108891296386719
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,7,0.01621759980916977
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,7,0.015294399857521058
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,7,0.01584800034761429
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,7,0.01549919992685318
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,1,0.015164799988269806
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,32767,4.107859039306641
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,7,0.022942399978637694
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,1,0.015614399313926696
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,7,0.021675199270248413
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,7,0.021619200706481934
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,7,0.021699200570583343
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,7,0.022152000665664674
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,7,0.021631999313831328
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,1,0.021524800360202788
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,15,0.017166399955749513
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,15,0.01576640009880066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,15,0.01568160057067871
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,15,0.015412800014019012
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,15,0.015291200578212738
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,15,0.02520799934864044
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,15,0.024516800045967103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,15,0.023895999789237975
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,15,0.0236735999584198
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,15,0.023577600717544556
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,15,0.024241599440574645
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,15,0.02382880002260208
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,31,0.017844800651073457
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,31,0.015703999996185304
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,31,0.015910400450229643
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,31,0.01555519998073578
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,31,0.01557600051164627
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,31,0.015884800255298613
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,31,0.01533920019865036
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,31,0.028227201104164122
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,31,0.026846399903297423
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,31,0.026576000452041625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,7,0.01549600064754486
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,31,0.026438400149345398
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,31,0.026358398795127868
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,31,0.026601600646972656
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,31,0.026441600918769837
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,7,0.022448000311851502
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,63,0.016862399876117706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,63,0.01635040044784546
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,63,0.016334399580955505
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,15,0.015779200196266174
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,63,0.016433599591255187
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,15,0.016059200465679168
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,63,0.03536159992218017
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,63,0.0325984001159668
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,63,0.03246079981327057
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,63,0.032564800977706906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,63,0.033020800352096556
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,7,0.015697599947452547
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,63,0.03282879889011383
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,63,0.01828480064868927
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,1,0.021403199434280394
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,1,0.021328000724315642
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,127,0.01812800019979477
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,63,0.015928000211715698
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,127,0.01823360025882721
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,63,0.033657601475715636
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,127,0.01817920058965683
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,127,0.05490720272064209
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,127,0.04733920097351074
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,127,0.047004801034927365
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,127,0.04669919908046723
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,127,0.04725280106067657
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,127,0.04678559899330139
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,255,0.027846398949623107
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,255,0.02338239997625351
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,255,0.02199520021677017
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,255,0.021564799547195434
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,255,0.021358400583267212
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,255,0.02199999988079071
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,255,0.021704000234603883
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,255,0.08895360231399536
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,255,0.08658720254898071
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,255,0.08520640134811401
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,127,0.018505600094795228
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,255,0.08461440205574036
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,127,0.01807519942522049
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,255,0.08461440205574036
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,255,0.08433120250701905
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,511,0.043227198719978335
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,511,0.03922080099582672
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,511,0.03751200139522552
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,511,0.03629119992256165
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,511,0.035953599214553836
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,511,0.03636800050735474
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,511,0.03594079911708832
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,511,0.16072319746017455
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,511,0.15577919483184816
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,511,0.15467519760131837
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,511,0.1546463966369629
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,511,0.15433599948883056
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,511,0.15476800203323365
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,1023,0.060782402753829956
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,1023,0.05620319843292236
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,1023,0.05411199927330017
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,1023,0.05332800149917603
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,1023,0.05322880148887634
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,1023,0.05284320116043091
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,1023,0.05263839960098267
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,1023,0.2911247968673706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,1023,0.2884160041809082
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,127,0.020310400426387785
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,1023,0.28526721000671384
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,63,0.01639840006828308
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,255,0.08470559716224671
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,1023,0.2853584051132202
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,1023,0.28539040088653567
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,2047,0.09692639708518982
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,127,0.01807360053062439
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,2047,0.09031360149383545
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,127,0.048655998706817624
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,2047,0.08833600282669067
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,2047,0.08748160004615783
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,2047,0.08675680160522461
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,2047,0.08743519783020019
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,2047,0.5469615936279297
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,2047,0.5443024158477783
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,2047,0.5425712108612061
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,2047,0.5418367862701416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,4095,0.162390398979187
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,2047,0.5411856174468994
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,2047,0.5407040119171143
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,4095,0.15410239696502687
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,4095,0.1532896041870117
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,4095,0.1528591990470886
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,4095,0.15302879810333253
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,1023,0.2860447883605957
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,4095,0.15293920040130615
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,1023,0.2858464002609253
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,2047,0.08798879981040955
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,4095,1.054263973236084
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,511,0.15699520111083984
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,4095,1.051523208618164
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,4095,1.0507712364196777
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,4095,1.0501088142395019
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,8191,0.2970832109451294
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,4095,1.0501839637756347
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,8191,0.28499839305877683
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,2047,0.54061918258667
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,4095,1.0491040229797364
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,8191,0.2843231916427612
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,4095,0.1574496030807495
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,8191,0.285150408744812
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,8191,0.28455519676208496
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,8191,2.069203186035156
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,8191,2.0647695541381834
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,8191,2.067616081237793
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,8191,2.062860870361328
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,8191,0.28692638874053955
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,4095,1.0499183654785156
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,8191,0.2832063913345337
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,16383,0.5696159839630127
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,16383,0.5559391975402832
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,16383,0.5462672233581543
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,8191,2.0655824661254885
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,16383,0.5471407890319824
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,8191,2.06329288482666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,16383,0.5432208061218262
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,16383,0.5450607776641846
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,16383,0.5442431926727295
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,8191,2.0654975891113283
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,16383,4.098582458496094
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,16383,4.096126556396484
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,16383,4.095032119750977
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,1,0.025992000102996828
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,1,0.02343360036611557
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,16383,4.094659042358399
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,1,0.023171199858188628
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,1,0.022457599639892578
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,1,0.02269600033760071
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,1,0.02220800071954727
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,1,0.03223040103912354
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,1,0.02945919930934906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,1,0.028809601068496705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,1,0.0284527987241745
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,1,0.028352001309394838
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,1,0.02831200063228607
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,16383,4.095304107666015
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,3,0.025787198543548585
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,3,0.023289600014686586
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,3,0.022392000257968902
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,3,0.022702400386333466
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,3,0.022313599288463593
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,3,0.022364799678325654
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,3,0.030134400725364684
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,3,0.029104000329971312
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,3,0.028460800647735596
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,3,0.028654399514198303
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,3,0.02889760136604309
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,3,0.02879520058631897
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,7,0.025968000292778015
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,7,0.02351360023021698
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,7,0.023046399652957916
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,7,0.02250719964504242
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,7,0.02226399928331375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,7,0.022676800191402436
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,16383,4.096803283691406
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,7,0.03568960130214691
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,7,0.03164480030536652
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,7,0.03181920051574707
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,7,0.03099200129508972
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,7,0.03067359924316406
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,1,0.02228800058364868
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,7,0.030870398879051207
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,1,0.028799998760223388
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,15,0.023393599689006804
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,15,0.022599999606609345
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,15,0.022470399737358093
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,15,0.022606399655342103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,15,0.023171199858188628
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,15,0.0406143993139267
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,15,0.03444960117340088
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,15,0.034195199608802795
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,15,0.033425599336624146
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,15,0.033348798751831055
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,15,0.033513599634170534
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,15,0.03373120129108429
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,31,0.026811200380325317
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,31,0.02356480062007904
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,31,0.023340800404548646
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,31,0.022459200024604796
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,31,0.02250880002975464
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,31,0.02266719937324524
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,7,0.022283199429512023
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,31,0.022868800163269042
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,31,0.04862880110740662
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,31,0.04090720117092132
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,31,0.03994719982147217
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,31,0.03935680091381073
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,7,0.030726400017738343
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,31,0.03925119936466217
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,15,0.023391999304294586
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,31,0.03924480080604553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,63,0.032528001070022586
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,63,0.024596799910068513
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,63,0.022944000363349915
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,63,0.023057599365711213
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,63,0.023287999629974365
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,63,0.023438400030136107
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,63,0.06576640009880066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,63,0.05899519920349121
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,63,0.054420799016952515
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,63,0.05318880081176758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,63,0.05166400074958801
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,63,0.052236801385879515
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,63,0.052219200134277347
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,127,0.037985599040985106
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,127,0.034062400460243225
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,127,0.029495999217033386
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,127,0.027958399057388304
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,127,0.027188798785209654
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,127,0.02686559855937958
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,127,0.026982399821281432
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,127,0.10040479898452759
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,127,0.09572160243988037
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,16383,4.101612854003906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,127,0.09390400052070617
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,31,0.03923520147800445
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,127,0.09197760224342347
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,127,0.09123520255088806
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,127,0.09088960289955139
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,255,0.0473904013633728
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,63,0.02370239943265915
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,255,0.04168640077114105
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,255,0.04012320041656494
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,3,0.02210240066051483
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,255,0.03934240043163299
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,3,0.03253119885921478
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,255,0.165283203125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,255,0.1617616057395935
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,255,0.1588256001472473
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,255,0.15884319543838502
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,255,0.15798720121383666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,255,0.15788320302963257
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,127,0.09377599954605102
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,511,0.0665727972984314
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,511,0.06279199719429016
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,511,0.06096479892730713
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,511,0.06054239869117737
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,511,0.06042400002479553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,255,0.04343520104885101
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,511,0.30699360370635986
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,511,0.2997488021850586
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,255,0.03960959911346436
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,511,0.29612159729003906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,511,0.29431838989257814
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,255,0.16013439893722534
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,511,0.29373281002044677
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,511,0.293503999710083
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,1023,0.10915679931640625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,1023,0.10103520154953002
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,1023,0.09433280229568482
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,1023,0.09360640048980713
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,1023,0.09357119798660278
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,1023,0.0934000015258789
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,511,0.07529919743537902
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,1023,0.5630064010620117
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,1023,0.5548768043518066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,1023,0.5512991905212402
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,511,0.060089600086212155
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,15,0.025806400179862975
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,1023,0.5498000144958496
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,255,0.04005120098590851
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,2047,0.16648160219192504
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,2047,0.16313600540161133
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,2047,0.1594272017478943
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,2047,0.16012959480285643
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,2047,0.1586992025375366
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,511,0.2937743902206421
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,2047,0.15964159965515137
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,2047,1.0696080207824707
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,2047,1.0618368148803712
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,2047,1.058240032196045
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,1023,0.5502831935882568
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,1023,0.5492047786712646
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,2047,1.0557711601257325
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,2047,1.0548463821411134
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,2047,1.0551551818847655
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,4095,0.29817121028900145
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,4095,0.29213120937347414
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,1023,0.09702079892158508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,4095,0.29024319648742675
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,4095,0.2888079881668091
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,4095,0.2886672019958496
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,2047,0.17558720111846923
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,2047,1.056107234954834
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,4095,2.0853424072265625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,4095,2.0707088470458985
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,4095,0.30757439136505127
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,4095,0.2891455888748169
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,8191,0.6128608226776123
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,4095,2.071086311340332
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,4095,2.069112014770508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,8191,0.5621871948242188
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,4095,2.0701152801513674
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,4095,2.0689184188842775
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,8191,0.5757631778717041
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,1023,0.5496335983276367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,8191,0.5476528167724609
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,8191,0.5475696086883545
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,8191,0.5458240032196044
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,8191,0.5512639999389648
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,4095,2.075721549987793
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,8191,4.118513488769532
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,1,0.05101439952850342
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,8191,4.101956939697265
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,1,0.039873600006103516
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,1,0.038159999251365664
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,8191,4.100193786621094
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,1,0.03696160018444061
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,1,0.0367680013179779
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,8191,4.0969696044921875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,1,0.036616000533103946
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,1,0.036739200353622437
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,1,0.04891040027141571
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,1,0.044377601146698
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,1,0.043163201212882994
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,1,0.043219199776649474
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,3,0.05014879703521728
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,3,0.03987999856472015
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,3,0.03802559971809387
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,3,0.03743999898433685
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,3,0.037171199917793274
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,3,0.03689279854297638
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,3,0.03715200126171112
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,3,0.06055359840393067
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,3,0.050332802534103396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,3,0.046372801065444946
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,3,0.04588960111141205
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,3,0.04541760087013245
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,3,0.04514720141887665
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,3,0.04531520009040833
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,8191,4.097404861450196
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,8191,4.09692497253418
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,7,0.04031679928302765
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,7,0.03819040060043335
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,7,0.03731360137462616
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,1,0.04352000057697296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,7,0.03719840049743652
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,7,0.037215998768806456
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,1,0.04345119893550873
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,7,0.06421279907226562
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,7,0.0494623988866806
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,7,0.04835839867591858
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,7,0.04806880056858063
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,7,0.04801760017871857
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,15,0.05117279887199402
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,15,0.04010719954967499
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,15,0.03836320042610168
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,15,0.03768480122089386
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,15,0.03743360042572021
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,15,0.037231999635696414
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,15,0.03731679916381836
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,15,0.07109439969062806
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,15,0.0620959997177124
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,15,0.05640159845352173
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,15,0.05456960201263428
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,15,0.05397599935531616
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,15,0.05376160144805908
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,15,0.05380960106849671
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,31,0.05275999903678894
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,31,0.04267840087413788
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,1,0.05815359950065613
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,31,0.03842560052871704
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,31,0.03771199882030487
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,31,0.03725599944591522
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,31,0.03715679943561554
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,31,0.037345600128173825
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,7,0.05470240116119385
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,31,0.07751839756965637
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,31,0.07217119932174683
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,7,0.04796479940414429
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,31,0.06673920154571533
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,31,0.06573759913444518
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,31,0.06607679724693298
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,63,0.05712959766387939
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,63,0.04832319915294647
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,63,0.04481599926948547
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,63,0.041833600401878356
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,63,0.03976480066776276
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,63,0.03924480080604553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,63,0.03878880143165588
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,63,0.1197424054145813
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,63,0.11111040115356445
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,63,0.1041759967803955
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,63,0.10287840366363525
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,63,0.10259519815444947
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,63,0.10224000215530396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,127,0.06653439998626709
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,127,0.05823839902877807
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,127,0.05257440209388733
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,127,0.05489439964294433
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,127,0.05141119956970215
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,7,0.05087519884109497
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,127,0.050329601764678954
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,127,0.18442399501800538
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,127,0.17693599462509155
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,127,0.17328159809112548
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,127,0.17110879421234132
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,7,0.03719359934329987
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,127,0.1694175958633423
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,127,0.16866719722747803
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,127,0.1685520052909851
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,255,0.08391839861869813
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,255,0.07579839825630189
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,31,0.06846079826354981
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,255,0.07195039987564086
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,255,0.0699728012084961
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,255,0.06902080178260803
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,255,0.06811839938163758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,255,0.31261119842529295
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,255,0.3053312063217163
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,255,0.3019455909729004
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,63,0.10749919414520263
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,255,0.2989887952804565
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,255,0.29767038822174074
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,511,0.13909440040588378
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,511,0.1224287986755371
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,511,0.11488480567932129
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,511,0.11059039831161499
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,511,0.10902400016784668
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,511,0.10774400234222412
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,31,0.08649920225143433
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,511,0.10764479637145996
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,255,0.06740319728851318
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,511,0.5909759998321533
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,511,0.575654411315918
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,511,0.5677008152008056
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,255,0.30006721019744875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,511,0.5637856006622315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,511,0.562065601348877
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,511,0.5626399993896485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,511,0.560697603225708
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,1023,0.18859679698944093
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,8191,4.0977424621582035
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,1023,0.17562559843063355
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,1023,0.17428640127182007
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,1023,0.17265440225601197
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,1023,0.1885424017906189
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,1023,1.102302360534668
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,1023,1.0766143798828125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,1023,1.072697639465332
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,1023,1.0704959869384765
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,255,0.2980272054672241
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,1023,0.20653440952301025
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,127,0.05071359872817993
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,2047,0.3354543924331665
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,2047,0.32031519412994386
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,2047,0.3101151943206787
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,1023,1.0699024200439453
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,2047,0.30587201118469237
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,2047,0.30297439098358153
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,2047,0.30066399574279784
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,1023,1.084607982635498
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,2047,2.1125999450683595
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,2047,2.1001871109008787
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,1023,0.18097599744796752
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,2047,2.0898303985595703
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,2047,2.084452819824219
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,2047,2.085481643676758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,2047,0.3026319980621338
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,4095,0.5687039852142334
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,4095,0.601259183883667
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,4095,0.5823472023010254
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,4095,0.5637968063354493
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,2047,2.083790397644043
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,2047,2.0855487823486327
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,4095,0.566648006439209
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,4095,0.5600831985473633
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,4095,0.5808800220489502
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,1023,1.0684752464294434
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,1,0.08764799833297729
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,1,0.07193440198898315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,1,0.06386399865150452
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,1,0.06121439933776855
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,1,0.060545599460601805
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,1,0.06085439920425415
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,1,0.06087999939918518
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,1,0.09756640195846558
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,4095,4.14045295715332
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,1,0.07699999809265137
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,4095,4.123651123046875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,4095,4.118510437011719
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,1,0.07035679817199707
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,1,0.06944159865379333
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,4095,4.118600082397461
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,4095,4.113723373413086
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,1,0.06948320269584655
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,1,0.06972799897193908
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,3,0.08837440013885497
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,3,0.06128159761428833
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,3,0.06141279935836792
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,3,0.06421599984169006
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,3,0.07313119769096374
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,3,0.10116959810256958
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,3,0.07835839986801148
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,3,0.07333760261535645
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,3,0.08603839874267578
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,3,0.0719871997833252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,3,0.07170079946517945
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,7,0.0723743975162506
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,7,0.06532959938049317
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,7,0.061536002159118655
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,7,0.06128799915313721
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,7,0.060540801286697386
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,7,0.06066719889640808
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,1,0.08288000226020813
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,7,0.10969120264053345
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,7,0.09315840005874634
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,7,0.08691359758377075
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,7,0.08023359775543212
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,7,0.0777455985546112
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,4095,4.117027282714844
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,7,0.07807360291481018
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,15,0.09003199934959412
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,3,0.060489600896835326
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,3,0.060782402753829956
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,15,0.06563680171966553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,15,0.06151679754257202
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,15,0.061003202199935914
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,15,0.06098080277442932
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,15,0.06102399826049805
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,15,0.1252992033958435
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,15,0.10817279815673828
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,7,0.08896480202674865
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,15,0.10218080282211303
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,15,0.09848319888114929
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,15,0.0929423987865448
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,15,0.09243360161781311
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,15,0.09090719819068908
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,31,0.09223039746284485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,31,0.06912000179290771
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,31,0.06374239921569824
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,31,0.06164640188217163
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,31,0.061166399717330934
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,31,0.06085759997367859
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,31,0.1555456042289734
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,31,0.13880000114440919
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,31,0.13185759782791137
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,31,0.12765439748764038
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,31,0.12687360048294066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,31,0.1262704014778137
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,31,0.1257040023803711
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,4095,4.111723327636719
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,63,0.07532320022583008
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,15,0.07323679924011231
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,63,0.07123039960861206
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,63,0.06981279850006103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,63,0.06982719898223877
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,63,0.06885920166969299
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,3,0.07206400036811829
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,63,0.21852478981018067
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,63,0.19995039701461792
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,63,0.19315040111541748
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,63,0.18855040073394774
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,63,0.18797279596328736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,31,0.0749504029750824
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,63,0.18711520433425904
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,127,0.11687519550323486
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,127,0.09141119718551635
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,127,0.0870848000049591
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,127,0.08560479879379272
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,127,0.08495200276374817
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,127,0.08449440002441407
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,127,0.34608960151672363
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,63,0.10016640424728393
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,127,0.32109119892120364
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,63,0.08060479760169983
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,127,0.31719839572906494
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,7,0.07776960134506225
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,127,0.3135727882385254
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,127,0.313700795173645
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,255,0.12635680437088012
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,255,0.12164640426635742
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,255,0.11954560279846191
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,255,0.11866079568862915
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,255,0.118286395072937
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,127,0.10031520128250122
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,255,0.598529577255249
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,255,0.5840832233428955
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,127,0.3293488025665283
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,255,0.5770480155944824
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,255,0.572057580947876
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,255,0.5697648048400878
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,511,0.25389599800109863
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,127,0.313974404335022
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,511,0.22135519981384277
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,255,0.14750880002975464
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,511,0.20725440979003906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,511,0.1960576057434082
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,511,0.19446560144424438
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,63,0.18932319879531861
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,511,1.1472864151000977
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,511,1.1187536239624023
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,511,1.100102424621582
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,255,0.5730800151824951
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,255,0.5700607776641846
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,511,1.0910271644592284
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,511,1.0897520065307618
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,255,0.13233599662780762
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,511,0.19903359413146973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,511,1.0899087905883789
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,1023,0.33951520919799805
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,1023,0.3857167959213257
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,1023,0.33075358867645266
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,1023,0.3276527881622314
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,1023,0.32572801113128663
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,1023,0.32386078834533694
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,1023,2.163742446899414
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,511,1.0933584213256835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,1023,2.1177663803100586
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,1023,2.1089408874511717
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,511,0.19572319984436035
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,1023,0.35532000064849856
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,2047,0.6779615879058838
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,2047,0.6211952209472656
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,2047,0.5986752033233642
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,1023,2.108318328857422
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,1023,2.1029647827148437
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,1023,2.105193519592285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,2047,0.6012063980102539
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,2047,0.583129596710205
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,1023,2.1337440490722654
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,2047,0.5884863853454589
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,2047,0.5813488006591797
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,2047,4.158422470092773
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,2047,4.146476745605469
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,1,0.013364799320697784
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,1,0.012860800325870513
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,1,0.012179200351238251
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,1,0.012110400199890136
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,1,0.011979199945926666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,1,0.012145599722862244
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,2047,4.139507293701172
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,1,0.018598400056362152
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,1,0.018824000656604768
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,1,0.017953599989414214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,1,0.01777759939432144
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,1,0.01725119948387146
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,3,0.01295360028743744
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,3,0.01271360069513321
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,3,0.012347199767827988
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,3,0.012243200093507767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,3,0.011923199892044068
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,3,0.012169600278139115
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,3,0.012017600238323212
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,3,0.01875839978456497
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,3,0.018078400194644927
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,3,0.017852799594402315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,3,0.01780160069465637
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,3,0.017683200538158417
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,3,0.01748320013284683
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,3,0.017262400686740877
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,7,0.013158400356769562
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,7,0.012809599936008453
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,2047,4.132619094848633
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,2047,4.189360046386719
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,7,0.012230399996042252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,7,0.012115199863910676
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,7,0.012160000205039979
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,2047,4.140449523925781
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,2047,4.135243225097656
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,7,0.018980799615383147
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,7,0.01786559969186783
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,7,0.01797119975090027
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,1,0.017948800325393678
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,7,0.017819200456142426
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,7,0.017825600504875184
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,15,0.013235199451446533
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,15,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,15,0.012945599853992462
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,15,0.012324800342321396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,15,0.012196800112724305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,15,0.012161599844694138
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,15,0.01204800009727478
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,15,0.0188960000872612
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,15,0.018323199450969697
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,15,0.017983999848365784
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,15,0.01770720034837723
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,15,0.017827199399471284
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,15,0.01775519996881485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,15,0.017766399681568144
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,31,0.01297920048236847
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,31,0.013288000226020813
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,31,0.012318400293588638
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,31,0.012164799869060517
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,31,0.012011200189590454
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,31,0.012118399888277055
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,31,0.01855359971523285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,31,0.01855040043592453
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,31,0.018028800189495087
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,31,0.017803199589252472
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,31,0.017652800679206847
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,31,0.01759680062532425
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,31,0.017662400007247926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,63,0.013096000254154205
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,63,0.012863999605178833
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,1,0.01231359988451004
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,7,0.012171199917793274
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,63,0.012433599680662155
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,63,0.012243200093507767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,63,0.012353599816560746
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,63,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,63,0.018702399730682374
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,63,0.01873279958963394
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,63,0.018059200048446654
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,63,0.017795200645923614
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,63,0.017824000120162962
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,7,0.017734399437904357
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,1,0.017496000230312347
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,127,0.014539200067520141
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,127,0.014377599954605103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,127,0.01387999951839447
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,127,0.013795199990272521
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,127,0.013756799697875976
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,127,0.013739199936389923
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,31,0.012411200255155564
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,127,0.013886399567127228
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,127,0.020227199792861937
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,127,0.02027360051870346
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,127,0.019648000597953796
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,127,0.019200000166893005
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,127,0.019534400105476378
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,127,0.01937279999256134
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,127,0.019793599843978882
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,255,0.01770080029964447
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,255,0.017819200456142426
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,255,0.016945600509643555
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,255,0.017056000232696534
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,255,0.016673600673675536
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,255,0.01685280054807663
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,255,0.016737599670886994
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,255,0.023339200019836425
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,255,0.02311359941959381
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,255,0.022921599447727203
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,7,0.0123648002743721
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,255,0.022657600045204163
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,255,0.022672000527381896
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,255,0.022707200050354003
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,511,0.018824000656604768
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,511,0.018961599469184874
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,511,0.01764480024576187
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,511,0.016590400040149687
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,63,0.012460800260305405
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,511,0.01744319945573807
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,511,0.018108800053596497
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,511,0.025412800908088683
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,511,0.02505599856376648
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,511,0.02359039932489395
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,511,0.023496000468730925
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,7,0.018464000523090364
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,511,0.02399519979953766
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,511,0.023664000630378722
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,63,0.017880000174045563
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,1023,0.019062399864196777
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,63,0.017956799268722533
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,1023,0.019731199741363524
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,1023,0.017126399278640746
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,1023,0.016926400363445282
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,1023,0.01799360066652298
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,1023,0.025772801041603087
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,1023,0.026446399092674256
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,1023,0.024403199553489685
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,1023,0.02375359982252121
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,1023,0.023212799429893495
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,1023,0.02406720072031021
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,1023,0.024372799694538115
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,2047,0.02126079946756363
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,2047,0.019974400103092194
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,2047,0.018764799833297728
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,2047,0.018115200102329254
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,2047,0.017484800517559053
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,2047,0.017844800651073457
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,2047,0.018166400492191315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,2047,0.028126400709152222
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,2047,0.02943359911441803
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,2047,0.027055999636650084
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,2047,0.02621600031852722
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,2047,0.026111999154090883
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,2047,0.026337599754333495
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,2047,0.026374399662017822
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,4095,0.023263999819755556
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,4095,0.022071999311447144
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,4095,0.020684799551963805
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,4095,0.01982560008764267
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,4095,0.02067359983921051
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,4095,0.020345599949359895
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,4095,0.033236798644065854
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,4095,0.03391520082950592
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,4095,0.03183839917182922
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,4095,0.031148800253868104
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,4095,0.031760001182556154
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,4095,0.031651198863983154
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,511,0.022467200458049775
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,8191,0.025931200385093688
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,8191,0.024719999730587007
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,8191,0.023574399948120116
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,8191,0.02215999960899353
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,8191,0.021985599398612977
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,1023,0.017956799268722533
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,1023,0.017862400412559508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,8191,0.04182400107383728
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,8191,0.04291839897632599
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,8191,0.04028320014476776
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,8191,0.03930079936981201
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,8191,0.03907999992370605
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,8191,0.04065600037574768
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,8191,0.04082399904727936
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,16383,0.030774399638175964
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,16383,0.02971999943256378
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,16383,0.026940798759460448
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,16383,0.025998398661613464
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,16383,0.025155198574066163
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,16383,0.025630399584770203
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,16383,0.025950399041175843
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,16383,0.06571199893951415
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,255,0.02258719950914383
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,16383,0.0628495991230011
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,16383,0.05697600245475769
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,16383,0.05525919795036316
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,16383,0.05459039807319641
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,16383,0.05454239845275879
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,511,0.01663679927587509
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,16383,0.05517759919166565
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,32767,0.039875200390815733
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,4095,0.03067359924316406
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,32767,0.032446399331092834
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,32767,0.03060320019721985
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,32767,0.031095999479293823
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,32767,0.029854398965835572
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,32767,0.10228960514068604
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,32767,0.10003679990768433
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,32767,0.09640799760818482
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,32767,0.09604480266571044
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,8191,0.023503999412059783
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,8191,0.024372799694538115
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,32767,0.09499359726905823
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,32767,0.09376959800720215
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,65535,0.05497440099716187
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,65535,0.04799999892711639
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,65535,0.045265600085258484
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,65535,0.043156799674034116
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,65535,0.04262720048427582
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,65535,0.17331039905548096
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,65535,0.16461280584335328
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,65535,0.16363519430160522
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,65535,0.16321120262145997
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,65535,0.1624079942703247
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,65535,0.16193439960479736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,65535,0.16237759590148926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,32767,0.037596800923347475
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,32767,0.030484798550605773
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,131071,0.06707519888877869
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,131071,0.062491202354431154
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,131071,0.062193602323532104
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,131071,0.06208320260047913
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,131071,0.06016160249710083
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,32767,0.0963808000087738
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,65535,0.04244639873504639
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,131071,0.29656639099121096
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,131071,0.3146687984466553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,131071,0.2874144077301025
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,131071,0.2930864095687866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,131071,0.28613760471343996
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,131071,0.2918848037719727
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,1,0.012904000282287598
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,1,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,1,0.01212640032172203
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,1,0.011910399794578553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,4095,0.01894560009241104
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,1,0.01886560022830963
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,1,0.018302400410175324
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,1,0.017612800002098083
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,1,0.017611199617385866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,131071,0.08371359705924988
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,1,0.0176816001534462
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,1,0.017416000366210938
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,1,0.01767839938402176
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,3,0.013126400113105775
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,3,0.012747199833393097
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,3,0.012216000258922577
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,3,0.012027200311422348
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,3,0.012049599736928939
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,3,0.012055999785661697
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,3,0.01875839978456497
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,3,0.01810240000486374
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,3,0.017763200402259826
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,65535,0.04241119921207428
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,3,0.017552000284194947
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,3,0.017367999255657195
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,131071,0.29193921089172364
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,7,0.013142399489879608
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,7,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,7,0.012179200351238251
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,7,0.01191840022802353
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,7,0.012025599926710128
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,1,0.01239679977297783
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,7,0.012123200297355651
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,7,0.01870719939470291
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,1,0.012015999853610992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,7,0.017612800002098083
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,7,0.017532800137996674
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,7,0.017539200186729432
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,7,0.017452800273895265
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,7,0.017576000094413756
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,15,0.0134320005774498
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,15,0.012956799566745758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,15,0.012392000108957291
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,131071,0.06051040291786194
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,3,0.012054400146007537
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,15,0.012331199645996094
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,15,0.012080000340938568
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,15,0.012244799733161926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,15,0.01204639971256256
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,15,0.01884319931268692
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,15,0.018326400220394133
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,15,0.017921599745750427
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,15,0.017731200158596038
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,15,0.017771199345588684
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,3,0.017585599422454835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,15,0.01762080043554306
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,31,0.013315199315547943
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,31,0.013033600151538849
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,31,0.01242400035262108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,31,0.012300799787044524
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,31,0.012006399780511856
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,31,0.012270399928092956
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,31,0.011959999799728394
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,31,0.018785600364208222
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,31,0.018086400628089905
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,31,0.017803199589252472
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,31,0.01770240068435669
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,31,0.01764799952507019
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,1,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,31,0.017587199807167053
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,63,0.01329279989004135
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,63,0.013059200346469879
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,63,0.012488000094890594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,63,0.012321600317955017
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,7,0.011990399658679962
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,63,0.012148799747228623
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,63,0.012127999961376191
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,63,0.018742400407791137
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,63,0.018296000361442567
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,63,0.018092800676822663
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,7,0.018145599961280824
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,63,0.017764799296855927
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,63,0.017534400522708892
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,63,0.01758880019187927
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,127,0.01496960073709488
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,127,0.014635199308395385
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,127,0.014139199256896972
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,127,0.013766400516033173
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,127,0.013716800510883332
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,127,0.013739199936389923
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,127,0.020505599677562714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,127,0.019942399859428406
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,127,0.01950719952583313
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,127,0.01926400065422058
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,127,0.01913280040025711
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,127,0.019254399836063384
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,127,0.0190528005361557
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,255,0.01812800019979477
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,255,0.017763200402259826
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,255,0.01730239987373352
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,255,0.017081600427627564
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,255,0.016976000368595125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,255,0.016873599588871004
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,255,0.016888000071048737
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,255,0.023494400084018707
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,255,0.02314079999923706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,31,0.01757279932498932
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,255,0.022784000635147093
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,255,0.022734400629997254
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,255,0.022438399493694305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,255,0.022759999334812164
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,255,0.022383999824523926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,63,0.012166400253772736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,511,0.019595199823379518
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,511,0.01896799951791763
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,511,0.01764959990978241
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,511,0.016838400065898894
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,511,0.01764799952507019
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,63,0.017324799299240114
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,511,0.02622239887714386
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,511,0.025227200984954835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,511,0.023848000168800353
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,511,0.023321600258350374
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,511,0.022889600694179536
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,511,0.024160000681877136
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,1023,0.02022559940814972
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,1023,0.019361600279808044
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,1023,0.0174127995967865
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,1023,0.017054399847984313
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,1023,0.017688000202178956
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,1023,0.018156799674034118
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,1023,0.02911840081214905
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,1023,0.028224000334739686
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,1023,0.026531198620796205
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,1023,0.025707200169563293
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,15,0.017689600586891174
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,1023,0.025830399990081788
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,1023,0.026067200303077697
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,3,0.01740480065345764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,2047,0.020236800611019134
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,2047,0.021622399985790252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,2047,0.019057600200176238
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,2047,0.01845120042562485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,2047,0.017951999604701997
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,2047,0.01814880073070526
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,2047,0.01839520037174225
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,2047,0.033188799023628236
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,511,0.017078399658203125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,2047,0.03144319951534271
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,511,0.01799360066652298
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,2047,0.029246398806571962
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,2047,0.029047998785972595
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,2047,0.02922239899635315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,2047,0.02945919930934906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,127,0.014006400108337402
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,4095,0.023108799755573273
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,511,0.02431679964065552
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,4095,0.019601599872112276
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,1023,0.01793760061264038
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,4095,0.020206399261951447
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,4095,0.0203232005238533
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,4095,0.042222398519515994
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,4095,0.03982880115509033
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,4095,0.03799999952316284
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,4095,0.0368800014257431
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,4095,0.03754720091819763
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,4095,0.037545600533485414
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,8191,0.029145601391792297
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,1023,0.025086399912834168
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,8191,0.025648000836372375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,8191,0.02407200038433075
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,8191,0.022700800001621245
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,8191,0.02544800043106079
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,8191,0.025935998558998107
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,8191,0.06362079977989196
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,8191,0.05892000198364258
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,8191,0.05454720258712768
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,8191,0.05216320157051087
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,2047,0.030048000812530517
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,8191,0.0517632007598877
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,8191,0.055555200576782225
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,8191,0.05500320196151733
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,4095,0.024784000217914583
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,16383,0.03921439945697784
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,16383,0.03473280072212219
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,16383,0.030003198981285097
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,16383,0.027585598826408386
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,4095,0.020729599893093108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,16383,0.026815998554229736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,16383,0.02884640097618103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,16383,0.029505598545074462
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,16383,0.09595999717712403
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,16383,0.09242240190505982
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,16383,0.09161919951438904
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,16383,0.09372000098228454
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,4095,0.037201601266860965
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,32767,0.05419679880142212
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,32767,0.039315199851989745
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,32767,0.0382752001285553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,32767,0.037006399035453795
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,32767,0.03925760090351105
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,32767,0.03951680064201355
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,8191,0.022951999306678773
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,32767,0.17257920503616334
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,32767,0.16251039505004883
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,32767,0.15952639579772948
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,32767,0.15856640338897704
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,32767,0.1596384048461914
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,32767,0.16074719429016113
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,65535,0.06314240097999572
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,65535,0.058633601665496825
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,65535,0.05687360167503357
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,65535,0.0561024010181427
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,4095,0.020401600003242492
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,65535,0.05816159844398498
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,65535,0.057359999418258666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,16383,0.09104639887809754
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,65535,0.31282720565795896
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,65535,0.292248010635376
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,16383,0.09467520117759705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,32767,0.045096001029014586
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,65535,0.2890383958816528
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,65535,0.2883631944656372
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,65535,0.2903696060180664
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,131071,0.13917920589447022
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,131071,0.0976415991783142
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,131071,0.09616960287094116
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,131071,0.09187840223312378
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,131071,0.09002559781074523
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,131071,0.09501439929008484
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,131071,0.09295200109481812
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,32767,0.15700639486312867
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,131071,0.5886528015136718
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,131071,0.5470320224761963
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,65535,0.08176640272140503
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,131071,0.545908784866333
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,1,0.013299199938774108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,131071,0.545956802368164
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,131071,0.5464608192443847
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,1,0.012563200294971466
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,1,0.012350399792194367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,1,0.012144000083208085
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,1,0.012160000205039979
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,16383,0.10048799514770508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,1,0.018904000520706177
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,1,0.01870719939470291
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,1,0.018192000687122345
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,65535,0.2899343967437744
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,1,0.01781280040740967
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,1,0.017759999632835387
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,1,0.01764959990978241
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,1,0.017671999335289002
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,3,0.013340799510478974
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,3,0.012883199751377106
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,3,0.012542399764060973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,3,0.01239679977297783
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,3,0.012294399738311767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,3,0.012272000312805176
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,3,0.012276799976825714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,3,0.01860959976911545
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,3,0.019124799966812135
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,3,0.01817920058965683
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,3,0.017923200130462648
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,3,0.01774719953536987
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,3,0.017668800055980684
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,3,0.017880000174045563
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,7,0.013417600095272065
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,7,0.013100799918174744
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,7,0.01244800016283989
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,7,0.012459199875593185
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,7,0.012204799801111221
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,7,0.012318400293588638
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,131071,0.5508624076843261
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,7,0.018907199800014495
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,7,0.01818079948425293
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,7,0.017875200510025023
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,7,0.017822399735450745
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,7,0.01777919977903366
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,7,0.017763200402259826
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,15,0.013441599905490875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,131071,0.5445615768432617
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,15,0.013128000497817992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,1,0.012108799815177918
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,15,0.01242400035262108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,15,0.012579199671745301
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,15,0.012127999961376191
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,15,0.012228800356388092
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,15,0.012150400131940842
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,15,0.01911199986934662
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,15,0.018688000738620758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,15,0.01810240000486374
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,15,0.017926399409770966
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,15,0.01791519969701767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,15,0.018011200428009033
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,31,0.013358399271965027
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,31,0.013156799972057343
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,31,0.012620800733566284
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,31,0.012455999851226807
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,31,0.012179200351238251
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,31,0.012432000041007996
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,31,0.012195199728012085
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,31,0.019016000628471374
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,31,0.01855839937925339
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,31,0.018134400248527527
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,31,0.01796479970216751
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,31,0.017766399681568144
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,31,0.01789119988679886
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,31,0.017611199617385866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,63,0.013265599310398103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,63,0.013030399382114411
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,63,0.012566399574279786
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,63,0.012518399953842163
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,63,0.012303999811410903
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,63,0.012435200065374375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,63,0.012411200255155564
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,63,0.01900160014629364
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,63,0.018926399946212768
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,63,0.018089599907398224
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,63,0.018203200399875642
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,63,0.01786080002784729
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,63,0.01786399930715561
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,63,0.018027199804782866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,127,0.014947199821472168
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,127,0.014958399534225463
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,127,0.014281600713729858
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,127,0.0142752006649971
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,127,0.01406400054693222
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,127,0.014071999490261078
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,127,0.01419519931077957
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,127,0.020798400044441223
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,127,0.020630399882793426
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,127,0.020175999402999877
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,127,0.019934399425983428
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,127,0.019835199415683746
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,127,0.019819200038909912
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,127,0.01972000002861023
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,255,0.018104000389575957
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,255,0.017884799838066102
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,255,0.0174127995967865
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,1,0.01286720037460327
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,255,0.017158399522304534
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,255,0.016967999935150146
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,255,0.01709440052509308
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,255,0.02398719936609268
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,255,0.023878400027751923
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,255,0.02353599965572357
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,255,0.02335360050201416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,15,0.01790879964828491
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,255,0.022963200509548188
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,255,0.023307199776172637
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,255,0.02311519980430603
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,511,0.020244799554347992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,511,0.01930239945650101
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,511,0.01801919937133789
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,511,0.01701759994029999
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,511,0.017367999255657195
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,511,0.01800000071525574
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,511,0.028812798857688903
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,511,0.028207999467849732
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,511,0.026787200570106508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,511,0.025916799902915955
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,65535,0.2910304069519043
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,511,0.02643679976463318
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,511,0.026635199785232544
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,1023,0.022067199647426605
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,1023,0.019871999323368073
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,1023,0.0185248002409935
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,1023,0.017985600233078002
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,1023,0.01740480065345764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,1023,0.01810079962015152
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,1023,0.018382400274276733
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,1023,0.033108800649642944
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,1023,0.029569599032402038
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,7,0.01218239963054657
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,1023,0.028307199478149414
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,7,0.018385599553585052
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,1023,0.02925119996070862
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,1023,0.02953920066356659
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,2047,0.024537600576877594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,2047,0.021673600375652313
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,2047,0.019964799284934998
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,2047,0.01892320066690445
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,2047,0.018329599499702455
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,2047,0.01913280040025711
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,2047,0.018952000141143798
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,255,0.0171424001455307
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,2047,0.0417136013507843
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,2047,0.036852800846099855
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,2047,0.03622879981994629
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,2047,0.03559519946575165
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,2047,0.03615680038928985
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,2047,0.03617280125617981
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,4095,0.029080000519752503
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,4095,0.024784000217914583
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,4095,0.022732800245285033
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,4095,0.021272000670433045
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,4095,0.020975999534130096
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,4095,0.022163200378417968
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,4095,0.022808000445365906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,4095,0.06366559863090515
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,511,0.018187199532985688
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,4095,0.05310239791870117
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,4095,0.050942397117614745
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,4095,0.04974240064620972
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,511,0.025518399477005006
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,4095,0.05146560072898865
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,8191,0.039083200693130496
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,8191,0.031814399361610415
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,8191,0.027595201134681703
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,8191,0.025747200846672057
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,8191,0.02584800124168396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,8191,0.028987199068069458
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,8191,0.030272001028060914
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,1023,0.031231999397277832
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,8191,0.10044159889221191
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,8191,0.09522079825401306
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,8191,0.08978239893913269
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,1023,0.02881920039653778
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,8191,0.09043840169906617
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,8191,0.0924560010433197
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,8191,0.09343360066413879
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,16383,0.05421280264854431
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,16383,0.04437440037727356
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,16383,0.037334400415420535
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,16383,0.03728480041027069
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,16383,0.03993920087814331
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,16383,0.04133439958095551
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,16383,0.17280800342559816
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,16383,0.16090879440307618
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,2047,0.03850559890270233
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,16383,0.15695040225982665
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,16383,0.15652480125427246
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,16383,0.15595840215682982
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,16383,0.1604383945465088
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,16383,0.16062239408493043
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,32767,0.061615997552871705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,32767,0.05797759890556335
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,32767,0.056041598320007324
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,32767,0.054688000679016115
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,32767,0.058075201511383054
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,4095,0.060140800476074216
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,32767,0.3115744113922119
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,32767,0.29203200340270996
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,4095,0.05196800231933594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,32767,0.2890304088592529
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,32767,0.2875328063964844
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,32767,0.2876143932342529
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,32767,0.29028480052947997
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,32767,0.29137439727783204
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,65535,0.09725919961929322
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,8191,0.0892960011959076
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,65535,0.0888975977897644
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,16383,0.03979040086269379
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,65535,0.09174559712409973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,65535,0.0924351990222931
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,65535,0.5882832050323487
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,65535,0.5477759838104248
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,32767,0.08089759945869446
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,65535,0.5448912143707275
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,32767,0.05918239951133728
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,65535,0.5432191848754883
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,65535,0.5396368026733398
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,131071,0.24803519248962402
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,65535,0.5459375858306885
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,65535,0.5467216014862061
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,131071,0.15846719741821289
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,131071,0.1560271978378296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,131071,0.15458719730377196
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,65535,0.13740479946136475
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,131071,0.15808320045471191
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,131071,0.15847040414810182
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,65535,0.09291679859161377
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,131071,1.1399200439453125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,131071,1.0582976341247559
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,131071,1.0530960083007812
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,131071,1.0522751808166504
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,1,0.014268800616264343
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,1,0.013048000633716583
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,1,0.0125231996178627
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,1,0.012435200065374375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,1,0.012227199971675873
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,1,0.012302400171756744
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,1,0.01228640004992485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,1,0.01967200040817261
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,1,0.01889120042324066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,1,0.01844000071287155
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,131071,1.055947208404541
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,1,0.017987200617790224
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,131071,1.055412769317627
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,1,0.01822720021009445
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,1,0.01796640008687973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,3,0.013123199343681335
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,3,0.013540799915790557
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,3,0.012457600235939026
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,3,0.012352000176906585
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,3,0.012532800436019897
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,3,0.019366399943828584
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,3,0.01914079934358597
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,3,0.018433600664138794
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,3,0.018244799971580506
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,3,0.01818400025367737
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,3,0.018243199586868285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,3,0.018118399381637573
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,7,0.013600000739097595
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,7,0.012868799269199371
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,7,0.01279519945383072
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,7,0.012294399738311767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,7,0.012540799379348756
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,65535,0.08980479836463928
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,7,0.012206400185823441
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,7,0.012569600343704223
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,7,0.018900799751281738
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,7,0.019497600197792054
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,7,0.01828159987926483
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,7,0.018054400384426118
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,7,0.017969599366188048
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,7,0.01803999990224838
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,15,0.01340479999780655
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,15,0.013118399679660797
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,15,0.01268640011548996
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,15,0.01241919994354248
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,15,0.012247999757528305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,15,0.012196800112724305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,15,0.012361600250005721
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,15,0.01934880018234253
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,15,0.018916800618171692
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,131071,1.0554047584533692
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,15,0.01826400011777878
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,15,0.017961600422859193
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,15,0.01812479943037033
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,15,0.017998400330543517
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,31,0.014577600359916686
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,31,0.012636800110340119
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,31,0.012406399846076966
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,31,0.012297599762678146
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,31,0.012342400103807449
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,31,0.01244800016283989
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,1,0.018294399976730345
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,31,0.019582399725914003
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,31,0.018747200071811677
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,31,0.018632000684738158
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,31,0.01807679980993271
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,31,0.018347199261188506
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,31,0.01808159947395325
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,31,0.018118399381637573
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,3,0.012455999851226807
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,63,0.013414399325847625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,63,0.013374400138854981
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,63,0.012748800218105316
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,63,0.012697599828243256
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,63,0.01244800016283989
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,63,0.012388800084590913
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,63,0.012460800260305405
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,63,0.02024960070848465
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,63,0.019315199553966524
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,63,0.018694399297237395
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,63,0.018563200533390046
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,63,0.018292799592018127
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,7,0.018544000387191773
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,63,0.01841920018196106
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,127,0.01499360054731369
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,127,0.014955200254917145
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,127,0.014572800695896148
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,127,0.014262400567531586
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,127,0.014248000085353851
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,127,0.014116799831390381
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,127,0.014241600036621093
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,127,0.021612800657749176
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,127,0.020640000700950623
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,15,0.018590399622917177
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,127,0.0204815998673439
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,127,0.020609599351882935
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,127,0.020155200362205507
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,127,0.020526400208473204
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,31,0.01311040073633194
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,255,0.017785599827766417
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,255,0.017608000338077544
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,255,0.01721920073032379
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,255,0.017289599776268004
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,255,0.01712159961462021
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,255,0.017324799299240114
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,255,0.02627519965171814
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,255,0.026075199246406555
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,255,0.025830399990081788
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,255,0.025308799743652344
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,255,0.025419199466705324
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,255,0.025227200984954835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,511,0.022166399657726286
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,3,0.012771199643611907
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,511,0.019966399669647215
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,131071,0.16305439472198485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,511,0.017664000391960144
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,511,0.018143999576568603
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,511,0.0172992005944252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,511,0.018668800592422485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,511,0.033667200803756715
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,511,0.02975359857082367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,511,0.028939199447631837
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,63,0.018537600338459016
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,511,0.02882240116596222
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,511,0.02985279858112335
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,511,0.029934400320053102
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,1023,0.025007998943328856
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,1023,0.02149759978055954
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,1023,0.019305600225925444
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,1023,0.01852000057697296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,1023,0.01910720020532608
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,1023,0.01897120028734207
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,1023,0.042931199073791504
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,127,0.021155199408531188
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,1023,0.037108799815177916
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,1023,0.03581919968128204
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,1023,0.035631999373435974
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,1023,0.03622399866580963
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,1023,0.03640959858894348
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,2047,0.029359999299049377
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,2047,0.02481440007686615
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,2047,0.021771200001239777
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,2047,0.020755200088024138
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,2047,0.020399999618530274
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,2047,0.02107519954442978
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,2047,0.021044799685478212
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,2047,0.06436480283737182
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,2047,0.0603983998298645
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,2047,0.05392000079154968
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,2047,0.05241439938545227
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,2047,0.05036479830741882
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,2047,0.05082880258560181
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,255,0.02537280023097992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,2047,0.05082079768180847
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,4095,0.03834240138530731
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,4095,0.03367519974708557
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,4095,0.02505599856376648
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,4095,0.02483679950237274
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,4095,0.027233600616455078
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,4095,0.10004160404205323
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,511,0.031563198566436766
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,4095,0.09403679966926574
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,4095,0.08936960101127625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,4095,0.09061440229415893
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,4095,0.0879967987537384
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,4095,0.09152960181236267
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,4095,0.09035360217094421
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,8191,0.05283679962158203
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,8191,0.04176320135593414
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,1023,0.0389488011598587
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,8191,0.036985599994659425
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,8191,0.03620319962501526
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,8191,0.04030239880084992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,255,0.018027199804782866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,8191,0.04134239852428436
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,8191,0.170415997505188
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,8191,0.1613711953163147
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,8191,0.1554095983505249
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,8191,0.15633759498596192
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,8191,0.15557440519332885
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,8191,0.16000319719314576
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,4095,0.02643199861049652
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,16383,0.0781216025352478
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,511,0.018408000469207764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,16383,0.061564797163009645
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,16383,0.05551519989967346
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,16383,0.05512319803237915
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,16383,0.05398719906806946
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,16383,0.05603839755058289
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,16383,0.05863519906997681
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,16383,0.30881600379943847
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,16383,0.29184319972991946
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,16383,0.289465594291687
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,16383,0.2869744062423706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,16383,0.2859711885452271
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,16383,0.29079360961914064
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,32767,0.1329360008239746
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,8191,0.03742400109767914
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,16383,0.29170238971710205
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,32767,0.08907359838485718
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,32767,0.0897711992263794
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,32767,0.08841919898986816
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,32767,0.09088000059127807
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,32767,0.09284639954566956
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,32767,0.5853936195373535
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,32767,0.5484928131103516
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,32767,0.5449359893798829
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,32767,0.5434016227722168
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,4095,0.026475200057029726
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,32767,0.5465807914733887
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,65535,0.16861759424209594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,32767,0.5472896099090576
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,65535,0.15879839658737183
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,65535,0.1565440058708191
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,65535,0.1544975996017456
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,1023,0.01820639967918396
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,65535,0.15846719741821289
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,65535,0.16081600189208983
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,32767,0.09701759815216064
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,65535,1.1309791564941407
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,65535,1.0546367645263672
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,65535,1.051375961303711
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,65535,1.0502464294433593
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,8191,0.16044479608535767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,131071,0.29916160106658934
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,131071,0.4629471778869629
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,65535,1.0560272216796875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,65535,1.0560704231262208
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,65535,0.2404320001602173
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,131071,0.28871679306030273
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,131071,0.2903680086135864
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,131071,0.29660959243774415
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,65535,1.05665922164917
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,131071,2.074468803405762
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,131071,2.217812728881836
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,131071,2.0700992584228515
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,32767,0.5422080039978028
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,1,0.014603200554847717
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,1,0.012716799974441528
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,1,0.012665599584579468
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,131071,0.2883055925369263
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,1,0.01266079992055893
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,1,0.0124719999730587
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,131071,0.2910559892654419
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,1,0.02040800005197525
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,1,0.018545599281787874
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,131071,2.067560005187988
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,1,0.018272000551223754
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,1,0.018111999332904815
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,1,0.018016000092029572
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,1,0.018006399273872375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,131071,2.0754512786865233
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,3,0.014699199795722961
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,131071,2.0762224197387695
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,3,0.012729600071907043
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,3,0.01295360028743744
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,3,0.01265919953584671
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,3,0.012452799826860428
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,3,0.0124719999730587
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,3,0.020203199982643128
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,3,0.019049599766731262
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,3,0.018089599907398224
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,3,0.018387199938297273
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,3,0.017947199940681457
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,3,0.018174399435520173
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,7,0.0149167999625206
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,7,0.013222399353981017
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,7,0.01268640011548996
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,7,0.012671999633312225
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,131071,2.0699792861938477
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,7,0.012439999729394913
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,7,0.012488000094890594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,7,0.012574400007724761
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,7,0.02022079974412918
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,7,0.0190080001950264
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,7,0.018249599635601042
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,7,0.018222400546073915
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,7,0.018160000443458557
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,1,0.013923199474811554
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,7,0.017975999414920805
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,15,0.014696000516414643
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,15,0.013260799646377563
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,15,0.012809599936008453
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,15,0.012572799623012543
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,15,0.012772800028324127
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,15,0.012542399764060973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,15,0.02048639953136444
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,15,0.01889919936656952
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,15,0.018572799861431122
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,1,0.01894560009241104
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,15,0.017982399463653563
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,15,0.018027199804782866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,15,0.01815039962530136
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,31,0.014903999865055084
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,31,0.013412800431251527
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,31,0.012889599800109864
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,31,0.012779200077056884
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,31,0.012408000230789185
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,3,0.018641600012779237
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,31,0.012777599692344665
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,31,0.01244800016283989
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,31,0.020499199628829956
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,31,0.019131200015544893
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,31,0.01902720034122467
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,31,0.018433600664138794
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,31,0.0182559996843338
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,31,0.018441599607467652
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,31,0.018334400653839112
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,63,0.014953599870204925
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,63,0.013406400382518769
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,63,0.013316799700260163
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,63,0.013016000390052795
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,63,0.012809599936008453
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,63,0.012664000689983367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,63,0.012580800056457519
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,63,0.020606400072574617
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,63,0.020267200469970704
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,7,0.018003199994564057
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,63,0.019424000382423402
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,63,0.01867839992046356
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,63,0.01870719939470291
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,63,0.01886879950761795
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,127,0.016510400176048278
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,15,0.01265760064125061
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,127,0.014852799475193024
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,127,0.014428800344467163
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,127,0.014455999433994293
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,127,0.0144896000623703
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,127,0.02417760044336319
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,127,0.023187200725078582
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,127,0.022448000311851502
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,15,0.018454399704933167
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,127,0.022383999824523926
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,127,0.022510400414466857
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,127,0.022300800681114195
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,3,0.013198399543762207
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,255,0.019169600307941438
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,255,0.017716799676418305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,255,0.018219199776649476
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,255,0.01807360053062439
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,255,0.01751199960708618
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,255,0.017350399494171144
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,255,0.030049601197242738
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,255,0.02924000024795532
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,255,0.02861759960651398
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,255,0.028347200155258177
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,255,0.02808319926261902
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,255,0.0282368004322052
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,255,0.028086400032043456
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,511,0.026531198620796205
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,511,0.021891200542449953
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,511,0.019636799395084382
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,511,0.01845120042562485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,511,0.017961600422859193
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,511,0.019019199907779692
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,127,0.015124799311161041
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,1,0.0124208003282547
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,511,0.03939839899539947
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,511,0.036817601323127745
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,127,0.014401599764823914
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,511,0.03517119884490967
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,511,0.035385599732398985
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,511,0.036313599348068236
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,511,0.03603360056877136
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,1023,0.03219200074672699
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,1023,0.025409600138664244
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,1023,0.020635199546813966
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,1023,0.019980800151824952
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,127,0.022388799488544463
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,1023,0.02102559953927994
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,1023,0.0208624005317688
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,1023,0.06632159948348999
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,1023,0.06017919778823853
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,1023,0.053260797262191774
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,255,0.019649599492549897
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,1023,0.05007359981536865
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,1023,0.04967359900474548
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,1023,0.049726399779319766
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,1023,0.04978879988193512
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,2047,0.040596801042556765
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,2047,0.03343999981880188
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,2047,0.026159998774528504
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,2047,0.024587200582027437
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,2047,0.023848000168800353
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,2047,0.02447039932012558
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,2047,0.024639999866485594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,2047,0.10225600004196167
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,2047,0.0949840009212494
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,2047,0.09160320162773132
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,2047,0.088782399892807
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,2047,0.08861280083656312
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,2047,0.0899728000164032
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,4095,0.05403839945793152
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,4095,0.04455519914627075
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,4095,0.03695200085639953
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,4095,0.036881598830223086
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,4095,0.03407999873161316
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,63,0.018801599740982056
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,4095,0.03791039884090423
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,4095,0.16103520393371581
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,511,0.019393600523471832
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,511,0.04508160054683685
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,4095,0.15644160509109498
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,4095,0.15538719892501832
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,4095,0.15425120592117308
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,4095,0.15635679960250853
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,4095,0.1568287968635559
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,1023,0.021264000236988066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,8191,0.0628495991230011
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,8191,0.056729602813720706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,8191,0.05392000079154968
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,8191,0.0528544008731842
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,8191,0.05782080292701721
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,8191,0.05811039805412292
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,8191,0.31007680892944334
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,8191,0.2921695947647095
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,8191,0.2875360012054443
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,8191,0.2857680082321167
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,8191,0.2854048013687134
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,16383,0.13603999614715576
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,2047,0.08979039788246154
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,16383,0.09626719951629639
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,16383,0.09133120179176331
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,16383,0.08827360272407532
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,16383,0.0874400019645691
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,16383,0.09135040044784545
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,16383,0.09371520280838012
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,4095,0.17292159795761108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,16383,0.585532808303833
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,16383,0.547654390335083
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,16383,0.544543981552124
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,16383,0.5418896198272705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,16383,0.5406879901885986
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,16383,0.5467184066772461
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,32767,0.242195200920105
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,16383,0.5472432136535644
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,32767,0.1626863956451416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,8191,0.2898384094238281
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,32767,0.15595999956130982
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,32767,0.15291680097579957
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,32767,0.1549343943595886
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,8191,0.29176959991455076
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,32767,0.15900640487670897
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,4095,0.036494401097297666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,32767,1.1296671867370605
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,32767,1.0574352264404296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,32767,1.050977611541748
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,32767,1.0498687744140625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,8191,0.08087040185928344
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,65535,0.45616798400878905
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,65535,0.2868623971939087
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,32767,1.0546784400939941
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,32767,1.0546624183654785
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,65535,0.28249759674072267
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,65535,0.2836496114730835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,65535,0.28872320652008054
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,65535,0.28836960792541505
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,32767,0.15883680582046508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,32767,1.0525712013244628
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,65535,2.0726800918579102
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,65535,2.2192224502563476
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,65535,2.0642559051513674
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,65535,2.0636480331420897
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,65535,0.29410879611968993
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,65535,2.0697343826293944
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,131071,0.5622591972351074
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,131071,0.5452032089233398
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,131071,0.9040783882141114
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,131071,0.5487264156341553
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,131071,0.5519375801086426
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,65535,2.0697711944580077
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,65535,2.0668304443359373
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,131071,0.5492752075195313
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,131071,0.5510704040527343
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,131071,4.104313659667969
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,131071,4.43505744934082
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,1,0.015246400237083435
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,131071,4.09992790222168
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,1,0.015014399588108063
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,131071,4.0984046936035154
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,1,0.013369600474834441
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,1,0.013315199315547943
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,1,0.013059200346469879
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,1,0.013291199505329133
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,1,0.013193599879741669
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,1,0.021080000698566435
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,1,0.02030719965696335
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,1,0.018952000141143798
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,1,0.01912800073623657
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,1,0.018857599794864656
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,3,0.014510400593280792
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,3,0.013603200018405915
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,3,0.01348000019788742
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,3,0.013264000415802002
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,131071,4.097268676757812
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,3,0.01343040019273758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,3,0.02136320024728775
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,3,0.012923200428485871
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,3,0.02011840045452118
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,3,0.019120000302791595
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,3,0.019139200448989868
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,3,0.019032000005245207
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,3,0.01870719939470291
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,3,0.019023999571800232
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,7,0.01520799994468689
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,7,0.014628799259662628
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,7,0.013444800674915314
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,7,0.013278399407863618
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,7,0.01327199935913086
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,7,0.013203200697898865
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,7,0.013409599661827087
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,7,0.021374399960041045
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,7,0.01936960071325302
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,7,0.019539199769496918
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,7,0.019047999382019044
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,7,0.01902880072593689
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,7,0.019105599820613862
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,15,0.015740799903869628
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,15,0.013780799508094788
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,15,0.013516800105571746
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,15,0.01462240070104599
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,131071,4.101816177368164
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,15,0.0135903999209404
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,15,0.013467200100421906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,15,0.021382400393486024
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,15,0.013376000523567199
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,15,0.01953279972076416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,1,0.01889120042324066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,15,0.019121600687503813
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,15,0.019518400728702544
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,15,0.019075199961662292
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,31,0.014793600142002105
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,31,0.013784000277519226
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,31,0.013398399949073792
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,31,0.013417600095272065
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,31,0.01321759968996048
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,31,0.01334560066461563
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,31,0.021796800196170807
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,31,0.020670400559902193
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,31,0.01959040015935898
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,31,0.019383999705314636
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,31,0.019607999920845033
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,31,0.019543999433517457
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,7,0.020110400021076204
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,63,0.015244799852371215
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,63,0.015003199875354766
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,63,0.013808000087738036
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,63,0.013542400300502777
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,63,0.013700799643993377
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,63,0.013838399946689606
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,63,0.013876800239086152
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,63,0.02340639978647232
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,63,0.022787199914455415
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,15,0.020230400562286376
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,1,0.01934719979763031
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,63,0.02155359983444214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,15,0.01902559995651245
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,63,0.02171040028333664
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,63,0.021550400555133818
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,63,0.021748800575733186
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,127,0.016828800737857818
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,127,0.016089600324630738
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,127,0.015515199303627015
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,127,0.015508800745010376
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,127,0.015652799606323244
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,127,0.02794399857521057
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,127,0.02727839946746826
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,131071,4.099571228027344
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,31,0.01942719966173172
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,127,0.026137599349021913
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,127,0.026396799087524413
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,127,0.026233598589897156
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,127,0.025961598753929137
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,255,0.020707200467586517
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,255,0.019923199713230134
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,255,0.01894560009241104
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,255,0.018644799292087556
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,255,0.018561600148677825
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,255,0.01868640035390854
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,255,0.01857759952545166
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,255,0.03668639957904816
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,127,0.017107200622558594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,255,0.03518719971179962
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,127,0.015430399775505066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,255,0.03505760133266449
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,255,0.034964799880981445
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,255,0.03510720133781433
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,511,0.035812801122665404
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,511,0.02775680124759674
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,511,0.022540800273418427
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,511,0.02091519981622696
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,511,0.020241600275039674
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,511,0.02144639939069748
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,511,0.021540799736976625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,511,0.07056159973144531
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,511,0.06278560161590577
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,511,0.054636800289154054
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,511,0.05126399993896484
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,511,0.05002560019493103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,511,0.050409597158432004
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,63,0.021671999990940095
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,511,0.0509984016418457
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,1023,0.043536001443862916
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,3,0.01571359932422638
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,31,0.015531200170516967
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,1023,0.023848000168800353
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,1023,0.02367199957370758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,1023,0.024027200043201448
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,255,0.034999999403953555
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,1023,0.10529119968414306
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,1023,0.09541760087013244
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,1023,0.09018080234527588
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,1023,0.08872320055961609
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,1023,0.08732159733772278
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,1023,0.08820319771766663
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,2047,0.05796639919281006
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,2047,0.04660159945487976
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,2047,0.03797760009765625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,2047,0.03705120086669922
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,2047,0.03670400083065033
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,2047,0.037136000394821164
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,127,0.026105600595474242
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,2047,0.17662080526351928
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,2047,0.16356159448623658
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,2047,0.15713119506835938
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,2047,0.15545120239257812
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,2047,0.15465760231018066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,2047,0.15516159534454346
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,2047,0.15544480085372925
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,1023,0.03540320098400116
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,4095,0.08561120033264161
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,255,0.03613280057907105
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,4095,0.05771840214729309
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,1023,0.024691200256347655
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,4095,0.05503360033035278
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,4095,0.055313599109649655
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,4095,0.3152031898498535
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,4095,0.2936496019363403
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,1023,0.08827199935913085
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,4095,0.2888063907623291
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,4095,0.28611040115356445
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,2047,0.03437440097332001
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,4095,0.28751039505004883
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,4095,0.2884848117828369
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,8191,0.1416416049003601
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,8191,0.09373279809951782
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,8191,0.08901439905166626
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,8191,0.08631680011749268
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,8191,0.09261599779129029
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,8191,0.5909759998321533
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,4095,0.06464959979057312
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,1023,0.027531200647354127
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,4095,0.053755199909210204
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,8191,0.5530288219451904
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,8191,0.546779203414917
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,8191,0.5381680011749268
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,8191,0.5474271774291992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,4095,0.28563361167907714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,16383,0.16780320405960084
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,16383,0.15862560272216797
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,16383,0.1548975944519043
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,16383,0.15319839715957642
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,8191,0.09223999977111816
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,16383,0.159552001953125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,16383,0.15943039655685426
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,4095,0.05339679718017578
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,16383,1.0586928367614745
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,16383,1.053326416015625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,8191,0.5415999889373779
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,16383,0.24606719017028808
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,16383,1.0491503715515136
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,16383,1.056105613708496
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,32767,0.4615664005279541
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,16383,1.054923152923584
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,32767,0.2967920064926147
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,32767,0.2893887996673584
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,32767,0.28382880687713624
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,32767,0.29029600620269774
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,32767,0.28972640037536623
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,16383,1.1360943794250489
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,16383,1.0512096405029296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,8191,0.09920480251312255
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,32767,2.221785545349121
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,32767,2.0693599700927736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,32767,0.2842303991317749
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,32767,2.064950370788574
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,32767,2.0639631271362306
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,65535,0.561352014541626
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,65535,0.9274751663208007
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,8191,0.5482751846313476
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,65535,0.5533088207244873
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,32767,2.070694351196289
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,32767,2.070356750488281
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,65535,0.5444960117340087
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,65535,0.5529551982879639
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,65535,0.5522064208984375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,32767,2.072857666015625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,65535,0.5463359832763672
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,1,0.016894400119781494
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,65535,4.106351852416992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,1,0.015806399285793304
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,65535,4.400073623657226
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,1,0.014123199880123139
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,1,0.014188799262046813
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,65535,4.096164703369141
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,1,0.014017599821090698
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,1,0.013713599741458892
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,1,0.020846399664878845
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,1,0.022168000042438508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,1,0.019566400349140166
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,1,0.019620800018310548
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,65535,4.102315139770508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,3,0.016524800658226015
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,3,0.015223999321460725
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,3,0.014449599385261535
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,3,0.01419519931077957
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,3,0.014000000059604644
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,3,0.014193600416183472
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,3,0.013831999897956849
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,3,0.022555199265480042
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,3,0.021481600403785706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,3,0.019870400428771973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,3,0.01966560035943985
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,3,0.019726400077342988
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,3,0.019705599546432494
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,3,0.019601599872112276
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,7,0.016327999532222748
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,7,0.015582400560379028
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,7,0.01412159949541092
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,7,0.01406240016222
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,1,0.013820800185203552
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,7,0.014097599685192109
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,7,0.013804799318313599
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,7,0.02253119945526123
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,1,0.020019200444221497
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,65535,4.099347305297852
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,7,0.02139520049095154
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,7,0.020547200739383698
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,1,0.01985439956188202
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,7,0.02009759992361069
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,7,0.019726400077342988
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,7,0.019923199713230134
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,15,0.016505600512027742
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,15,0.0157600000500679
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,15,0.01412159949541092
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,15,0.014107200503349304
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,15,0.014151999354362487
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,15,0.01414559930562973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,15,0.022495999932289124
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,15,0.021329599618911742
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,15,0.020611199736595153
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,15,0.020313599705696107
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,15,0.020206399261951447
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,15,0.020321600139141083
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,15,0.02022880017757416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,31,0.016607999801635742
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,31,0.015427200496196747
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,31,0.014451199769973755
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,31,0.01414559930562973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,31,0.014099200069904328
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,31,0.013935999572277069
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,31,0.014083200693130493
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,31,0.02476319968700409
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,31,0.023606400191783904
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,31,0.0225040003657341
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,31,0.022672000527381896
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,31,0.022120000422000886
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,31,0.022286400198936462
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,31,0.02216159999370575
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,63,0.01674399971961975
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,63,0.015907199680805208
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,63,0.014838400483131408
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,63,0.014591999351978302
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,63,0.014319999516010285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,63,0.014448000490665436
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,65535,4.101166534423828
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,65535,4.113169479370117
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,7,0.014008000493049622
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,63,0.027723199129104613
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,63,0.025648000836372375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,63,0.02687999904155731
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,63,0.02513279914855957
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,63,0.025260800123214723
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,63,0.025492799282073975
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,127,0.01836320012807846
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,1,0.019713599979877473
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,127,0.017961600422859193
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,127,0.01679839938879013
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,127,0.01613280028104782
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,127,0.016407999396324157
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,127,0.01615840047597885
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,127,0.036287999153137206
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,127,0.03421759903430939
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,127,0.03304319977760315
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,127,0.03284800052642822
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,127,0.032820799946784975
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,127,0.03304159939289093
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,127,0.03285120129585266
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,255,0.021265600621700288
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,255,0.02003999948501587
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,255,0.01958879977464676
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,255,0.01960480064153671
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,255,0.019644799828529357
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,255,0.019283199310302736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,255,0.05612800121307373
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,255,0.052051198482513425
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,255,0.049270400404930116
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,255,0.04825760126113891
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,255,0.048518401384353635
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,63,0.014620800316333771
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,255,0.0485071986913681
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,255,0.048825600743293764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,511,0.0367792010307312
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,511,0.03960959911346436
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,511,0.030691200494766237
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,511,0.02449280023574829
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,511,0.023814399540424348
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,511,0.024883200228214265
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,63,0.02515999972820282
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,7,0.01982239931821823
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,127,0.01669439971446991
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,511,0.1009552001953125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,511,0.09337279796600342
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,15,0.014526399970054626
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,511,0.08667839765548706
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,511,0.08884639739990234
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,511,0.09031999707221985
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,1023,0.05021439790725708
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,1023,0.051044797897338866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,1023,0.04301440119743347
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,1023,0.036499199271202085
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,1023,0.03784160017967224
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,1023,0.0385888010263443
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,255,0.02203039973974228
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,1023,0.16982560157775878
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,1023,0.16753920316696166
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,1023,0.16005760431289673
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,1023,0.1563680052757263
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,1023,0.15591360330581666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,1023,0.1565824031829834
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,1023,0.15745919942855835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,2047,0.06915199756622314
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,2047,0.06045600175857544
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,2047,0.05634880065917969
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,2047,0.05429279804229736
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,2047,0.054953598976135255
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,2047,0.05620480179786682
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,2047,0.3066864013671875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,2047,0.297326397895813
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,2047,0.2881839990615845
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,2047,0.28659520149230955
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,2047,0.28841280937194824
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,511,0.026209598779678343
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,511,0.09971359968185425
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,4095,0.13129440546035767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,4095,0.09527040123939515
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,511,0.09099680185317993
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,4095,0.08814560174942017
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,1023,0.037110400199890134
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,4095,0.09144480228424072
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,4095,0.0934719979763031
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,4095,0.581601619720459
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,4095,0.5476463794708252
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,4095,0.5435535907745361
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,2047,0.2913696050643921
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,4095,0.5444928169250488
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,4095,0.5491104125976562
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,2047,0.2882879972457886
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,8191,0.23926239013671874
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,4095,0.10436639785766602
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,8191,0.17193759679794313
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,4095,0.08977280259132385
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,8191,0.15612319707870484
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,8191,0.15389440059661866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,8191,0.1605232000350952
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,8191,0.16428799629211427
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,4095,0.5557295799255371
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,2047,0.07603679895401001
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,8191,1.126371192932129
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,8191,1.0574352264404296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,8191,1.0529824256896974
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,8191,1.0502047538757324
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,8191,1.0576319694519043
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,16383,0.30271520614624026
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,16383,0.46027522087097167
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,8191,0.16230560541152955
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,16383,0.291809606552124
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,16383,0.28805758953094485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,16383,0.28378241062164306
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,16383,0.2888767957687378
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,16383,0.2944672107696533
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,8191,1.0631199836730958
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,4095,0.5425936222076416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,16383,2.080713653564453
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,16383,2.0721456527709963
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,16383,2.06617431640625
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,16383,2.066182327270508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,16383,2.0710575103759767
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,32767,0.5674831867218018
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,32767,0.9030271530151367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,32767,0.5582015991210938
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,16383,2.0763727188110352
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,32767,0.554249620437622
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,32767,0.5497439861297607
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,32767,0.5519408226013184
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,32767,0.5524288177490234
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,16383,2.2129375457763674
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,8191,1.0608719825744628
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,1,0.02529120147228241
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,1,0.017552000284194947
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,1,0.015755200386047365
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,1,0.015447999536991119
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,1,0.015268799662590028
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,1,0.015385599434375763
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,1,0.015929600596427916
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,1,0.03192160129547119
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,1,0.023097600042819976
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,1,0.021555200219154358
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,1,0.021385599672794343
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,1,0.021406400203704833
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,1,0.02152000069618225
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,1,0.02125120013952255
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,3,0.025391998887062072
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,3,0.017103999853134155
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,3,0.016340799629688263
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,3,0.015492799878120422
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,32767,4.42564811706543
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,3,0.015251199901103973
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,3,0.01536799967288971
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,3,0.015852800011634825
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,32767,4.109468841552735
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,32767,4.096235275268555
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,3,0.02277279943227768
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,32767,4.1199806213378904
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,3,0.021953600645065307
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,3,0.021563200652599333
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,7,0.027118399739265442
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,7,0.016889600455760954
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,7,0.016043199598789214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,32767,4.106099319458008
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,7,0.015355199575424194
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,7,0.015748800337314607
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,7,0.015724800527095795
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,7,0.03341920077800751
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,7,0.023324799537658692
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,7,0.02213120013475418
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,7,0.02181120067834854
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,7,0.021779200434684752
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,7,0.021561600267887115
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,7,0.021660800278186797
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,15,0.025760000944137572
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,15,0.017371200025081635
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,15,0.016396799683570863
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,15,0.01571680009365082
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,3,0.03178719878196716
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,15,0.01565439999103546
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,15,0.015561600029468537
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,15,0.036051198840141296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,3,0.021488000452518464
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,3,0.02178879976272583
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,15,0.023707200586795808
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,15,0.023625600337982177
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,7,0.015611200034618378
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,15,0.023708799481391908
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,15,0.024139200150966645
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,31,0.025891199707984924
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,31,0.01732960045337677
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,31,0.0160303995013237
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,31,0.015515199303627015
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,31,0.015475200116634369
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,31,0.015540799498558045
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,31,0.015694400668144225
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,31,0.04178879857063293
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,31,0.028059199452400208
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,31,0.026873600482940675
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,31,0.02645280063152313
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,31,0.026344001293182373
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,31,0.026339200139045716
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,31,0.026572799682617186
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,63,0.028531199693679808
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,63,0.018267199397087097
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,63,0.01671680063009262
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,63,0.01624159961938858
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,32767,4.099395370483398
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,63,0.01583999991416931
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,15,0.015996800363063814
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,63,0.016574400663375854
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,63,0.016164800524711607
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,63,0.04872319996356964
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,63,0.0349727988243103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,15,0.025404798984527587
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,63,0.03271839916706085
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,3,0.021796800196170807
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,63,0.03293280005455017
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,127,0.03514719903469086
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,127,0.018878400325775146
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,127,0.01798879951238632
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,127,0.01817920058965683
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,127,0.01773280054330826
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,127,0.018164800107479097
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,127,0.0669983983039856
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,127,0.054343998432159424
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,127,0.05010719895362854
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,127,0.04758880138397217
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,127,0.0474368005990982
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,127,0.04736000001430511
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,127,0.047336000204086306
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,255,0.04240959882736206
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,255,0.027403199672698976
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,255,0.02268960028886795
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,255,0.022457599639892578
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,255,0.02239840030670166
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,255,0.022270399332046508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,255,0.10361599922180176
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,63,0.03321279883384705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,15,0.02444320023059845
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,255,0.08657280206680298
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,127,0.020164799690246583
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,255,0.08467360138893128
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,255,0.08450719714164734
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,255,0.08483520150184631
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,511,0.04279040098190308
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,511,0.039017599821090695
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,511,0.03683040142059326
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,511,0.03644959926605225
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,511,0.03620960116386414
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,511,0.03643839955329895
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,511,0.183843195438385
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,511,0.1586143970489502
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,255,0.02449920028448105
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,511,0.1570304036140442
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,32767,4.108160018920898
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,511,0.15533759593963622
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,511,0.15468159914016724
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,511,0.15470880270004272
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,63,0.03260639905929565
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,63,0.032393598556518556
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,1023,0.057436800003051756
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,255,0.08539999723434448
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,1023,0.053281599283218385
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,1023,0.053801602125167845
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,1023,0.05278559923171997
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,511,0.06324639916419983
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,1023,0.3211600065231323
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,1023,0.2814784049987793
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,1023,0.28597919940948485
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,1023,0.27909440994262696
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,1023,0.2851727962493896
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,1023,0.2785360097885132
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,2047,0.14463839530944825
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,2047,0.09476479887962341
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,2047,0.09055520296096801
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,2047,0.08748800158500672
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,2047,0.0872048020362854
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,1023,0.0899728000164032
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,1023,0.0612559974193573
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,1023,0.0541055977344513
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,2047,0.5946383953094483
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,2047,0.5481264114379882
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,1023,0.29080801010131835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,2047,0.5439136028289795
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,2047,0.5423408031463623
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,2047,0.5417935848236084
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,4095,0.16106079816818236
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,2047,0.08820319771766663
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,4095,0.1541632056236267
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,511,0.15919519662857057
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,4095,0.15366560220718384
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,4095,0.15245440006256103
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,4095,0.15297759771347047
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,2047,0.5413824081420898
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,2047,0.5413871765136719
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,4095,1.140321636199951
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,4095,0.25356640815734866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,4095,0.15645120143890381
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,4095,1.055820846557617
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,4095,1.0517999649047851
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,255,0.08963840007781983
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,4095,1.0498064041137696
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,4095,1.0494607925415038
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,4095,1.0492351531982422
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,8191,0.4722928047180176
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,8191,0.29311039447784426
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,8191,0.2851439952850342
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,8191,0.2836927890777588
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,8191,0.28511838912963866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,8191,0.28411839008331297
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,2047,0.08724640011787414
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,8191,2.070100784301758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,8191,2.067771148681641
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,8191,2.0664783477783204
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,4095,1.0509696006774902
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,16383,0.5775392055511475
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,8191,2.064169692993164
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,16383,0.5532351970672608
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,8191,2.0639215469360352
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,8191,2.062937545776367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,16383,0.5475520133972168
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,16383,0.549348783493042
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,16383,0.5470016002655029
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,8191,2.2333759307861327
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,8191,0.28631680011749266
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,16383,0.9172143936157227
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,16383,0.5488080024719239
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,16383,4.120091247558594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,16383,4.099348831176758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,16383,4.4167022705078125
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,16383,4.098905563354492
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,1,0.025916799902915955
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,1,0.02380640059709549
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,1,0.022977599501609804
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,1,0.022431999444961548
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,16383,4.097073745727539
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,1,0.02263039946556091
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,1,0.022121599316596983
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,1,0.05735359787940979
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,1,0.03246079981327057
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,1,0.02922559976577759
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,1,0.028603199124336242
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,1,0.0284496009349823
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,3,0.04832960069179535
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,3,0.025572800636291505
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,3,0.023416000604629516
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,3,0.023454399406909944
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,3,0.022038400173187256
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,3,0.0224031999707222
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,3,0.022124800086021423
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,16383,4.0944255828857425
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,3,0.032996800541877744
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,3,0.030035200715065002
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,3,0.028347200155258177
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,3,0.029126399755477907
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,3,0.028913599252700806
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,3,0.02847360074520111
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,7,0.04842880070209503
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,7,0.02327679991722107
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,7,0.02354079931974411
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,7,0.025947201251983642
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,7,0.02204640060663223
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,7,0.022617599368095397
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,7,0.0222463995218277
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,7,0.05862240195274353
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,7,0.03161759972572327
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,7,0.03146719932556152
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,7,0.0306768000125885
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,7,0.030811199545860292
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,7,0.030689600110054015
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,15,0.049425598978996274
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,15,0.02595199942588806
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,15,0.023451200127601622
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,15,0.022356800734996796
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,15,0.022310400009155275
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,15,0.02220800071954727
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,1,0.049456000328063965
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,15,0.060343998670578006
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,15,0.04049279987812042
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,15,0.034118399024009705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,15,0.03349440097808838
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,15,0.03340800106525421
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,15,0.033251199126243594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,1,0.02879360020160675
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,1,0.029227200150489806
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,31,0.027027198672294618
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,31,0.023892800509929656
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,31,0.02250239998102188
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,3,0.0564191997051239
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,31,0.02236959934234619
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,31,0.022379200160503387
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,31,0.06841599941253662
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,31,0.04876959919929504
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,31,0.0416159987449646
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,16383,4.110472106933594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,31,0.039656001329422
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,31,0.0391072005033493
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,31,0.03945760130882263
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,31,0.03901439905166626
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,63,0.053142398595809937
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,15,0.023691199719905853
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,63,0.024182400107383727
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,63,0.023625600337982177
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,63,0.02321600019931793
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,15,0.034483200311660765
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,63,0.023363199830055238
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,63,0.06591200232505798
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,63,0.05897120237350464
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,63,0.055297601222991946
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,31,0.04935519993305206
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,63,0.05373920202255249
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,63,0.05245440006256104
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,63,0.05240960121154785
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,31,0.0232464000582695
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,127,0.060185599327087405
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,127,0.03853760063648224
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,127,0.03369440138339996
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,127,0.02951360046863556
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,127,0.028542399406433105
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,127,0.02781279981136322
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,127,0.027617600560188294
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,127,0.10005760192871094
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,127,0.09358879923820496
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,127,0.09174879789352416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,7,0.035739201307296756
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,127,0.09131519794464112
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,127,0.09109280109405518
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,255,0.07303839921951294
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,255,0.047200000286102294
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,255,0.04244320094585419
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,255,0.04162879884243011
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,255,0.03973279893398285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,255,0.0397487998008728
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,255,0.03903039991855621
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,255,0.19104160070419313
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,255,0.16557279825210572
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,255,0.161899197101593
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,255,0.15973119735717772
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,63,0.0855072021484375
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,255,0.15823999643325806
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,255,0.15787839889526367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,511,0.11521439552307129
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,511,0.07561439871788025
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,511,0.06597440242767334
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,511,0.06333119869232177
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,511,0.05971999764442444
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,511,0.06061279773712158
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,511,0.059083199501037596
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,127,0.12227200269699097
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,511,0.3459552049636841
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,511,0.30634078979492185
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,511,0.2989808082580566
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,511,0.2937999963760376
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,511,0.2961855888366699
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,511,0.2929327964782715
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,511,0.2931391954421997
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,1023,0.16826720237731935
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,1023,0.1001471996307373
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,1023,0.09689279794692993
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,63,0.025038400292396547
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,1023,0.09437280297279357
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,1023,0.09331520199775696
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,1023,0.09348000288009643
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,255,0.15868959426879883
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,1023,0.5544479846954345
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,1023,0.5497360229492188
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,1023,0.5491504192352294
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,127,0.0964896023273468
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,1023,0.5485439777374268
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,2047,0.17536159753799438
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,2047,0.16677119731903076
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,2047,0.16240320205688477
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,2047,0.1591472029685974
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,63,0.03303839862346649
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,2047,0.15935519933700562
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,2047,0.15869439840316774
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,1023,0.6195263862609863
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,1023,0.5515247821807862
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,2047,1.1648863792419433
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,2047,1.0694239616394043
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,2047,1.0549712181091309
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,2047,0.27721118927001953
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,2047,1.0553199768066406
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,2047,1.0560239791870116
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,4095,0.3221215963363647
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,1023,0.11053600311279296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,4095,0.29195680618286135
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,4095,0.29723999500274656
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,4095,0.2894896030426025
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,4095,0.2886528015136719
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,1023,0.561575984954834
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,2047,1.0648688316345214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,2047,1.0602911949157714
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,4095,2.2585248947143555
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,4095,2.0843807220458985
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,4095,0.4937312126159668
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,4095,2.077190399169922
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,4095,2.0726240158081053
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,4095,0.2892303943634033
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,4095,2.069931221008301
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,8191,0.5892623901367188
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,8191,0.9286527633666992
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,8191,0.5507631778717041
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,8191,0.5799808025360107
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,4095,2.069811248779297
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,8191,0.5680607795715332
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,8191,0.5798431873321533
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,4095,2.0751087188720705
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,8191,0.5468639850616455
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,1,0.08847200274467468
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,8191,4.115446472167969
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,8191,4.106164932250977
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,1,0.04986560046672821
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,8191,4.452352142333984
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,1,0.03740000128746033
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,1,0.03983519971370697
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,1,0.036735999584198
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,8191,4.0974479675292965
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,1,0.05820639729499817
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,1,0.09653760194778442
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,1,0.044766399264335635
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,1,0.043515199422836305
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,1,0.04339680075645447
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,3,0.08645600080490112
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,3,0.04999040067195892
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,3,0.03999359905719757
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,3,0.03819999992847443
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,3,0.03764480054378509
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,8191,4.09784164428711
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,3,0.03697119951248169
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,3,0.036769598722457886
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,3,0.06143040060997009
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,3,0.05027040243148804
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,3,0.04628959894180298
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,3,0.04572640061378479
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,3,0.04531359970569611
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,8191,4.104377746582031
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,3,0.045263999700546266
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,1,0.0383679986000061
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,1,0.0369376003742218
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,7,0.08921279907226562
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,7,0.039955198764801025
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,7,0.03795199990272522
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,7,0.03742879927158356
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,7,0.03681119978427887
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,1,0.04366720020771027
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,7,0.06403359770774841
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,7,0.09908159971237182
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,7,0.05374240279197693
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,7,0.04802719950675964
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,7,0.047844800353050235
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,7,0.04797919988632202
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,15,0.05083199739456177
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,15,0.04091840088367462
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,15,0.03836480081081391
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,15,0.0373663991689682
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,15,0.03728800117969513
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,15,0.03722079992294312
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,3,0.09628639817237854
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,15,0.10729759931564331
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,15,0.07137600183486939
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,15,0.06252800226211548
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,8191,4.100516891479492
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,15,0.056011199951171875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,15,0.05404319763183594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,15,0.053881597518920896
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,31,0.09082880020141601
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,31,0.04300960004329681
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,31,0.05276319980621338
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,31,0.03852320015430451
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,31,0.037601599097251893
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,31,0.03731839954853058
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,31,0.03718880116939545
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,7,0.050672000646591185
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,31,0.12427840232849122
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,31,0.07762879729270936
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,31,0.07245919704437256
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,31,0.0690064013004303
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,31,0.06719040274620056
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,31,0.06682239770889283
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,63,0.0927839994430542
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,7,0.049323201179504395
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,63,0.04740639925003052
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,63,0.044627198576927186
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,63,0.041319999098777774
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,63,0.03925119936466217
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,63,0.038761600852012634
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,63,0.15489920377731323
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,63,0.11994080543518067
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,63,0.1086624026298523
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,63,0.10730880498886108
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,63,0.10452159643173217
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,63,0.1036687970161438
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,63,0.10249919891357422
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,15,0.054104000329971313
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,127,0.10641440153121948
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,127,0.06624320149421692
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,127,0.05716800093650818
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,127,0.05176159739494324
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,127,0.05115039944648743
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,127,0.05035039782524109
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,31,0.0864799976348877
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,127,0.22414560317993165
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,7,0.037110400199890134
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,127,0.17658400535583496
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,127,0.1731951951980591
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,63,0.05751519799232483
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,127,0.16905759572982787
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,15,0.08831679821014404
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,255,0.13232799768447875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,255,0.0843824028968811
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,255,0.07489600181579589
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,255,0.07218559980392455
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,255,0.06921600103378296
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,255,0.0688480019569397
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,255,0.0670687973499298
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,255,0.3602303981781006
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,255,0.31189119815826416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,255,0.3043168067932129
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,255,0.30148959159851074
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,127,0.054390400648117065
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,255,0.29949440956115725
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,255,0.29897279739379884
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,511,0.13992960453033448
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,511,0.21857600212097167
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,1,0.04916479885578155
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,511,0.12275840044021606
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,127,0.1847391963005066
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,511,0.11399519443511963
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,511,0.110481595993042
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,511,0.10914560556411743
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,127,0.17078080177307128
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,511,0.10767199993133544
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,127,0.16931999921798707
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,511,0.6661471843719482
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,511,0.5903359889984131
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,511,0.5686607837677002
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,511,0.5626751899719238
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,1023,0.18900959491729735
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,511,0.5619999885559082
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,1023,0.17590240240097046
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,1023,0.1747056007385254
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,1023,0.17325600385665893
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,255,0.29750399589538573
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,1023,1.2117839813232423
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,1023,1.0856656074523925
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,511,0.574883222579956
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,511,0.5634399890899658
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,1023,0.20523519515991212
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,1023,1.0768431663513183
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,1023,0.32270240783691406
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,1023,1.0706095695495605
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,1023,1.0717391967773438
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,1023,1.0702624320983887
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,2047,0.3392143964767456
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,2047,0.32762880325317384
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,2047,0.3068687915802002
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,2047,0.30251359939575195
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,2047,0.3032847881317139
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,1023,1.0996416091918946
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,1023,0.18076640367507935
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,2047,2.096892738342285
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,2047,2.1143520355224608
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,2047,2.092812728881836
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,2047,0.5483664035797119
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,2047,2.0834575653076173
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,4095,0.6312416076660157
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,4095,1.0245840072631835
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,4095,0.5697887897491455
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,4095,0.6219679832458496
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,2047,2.0841264724731445
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,2047,2.0824863433837892
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,4095,0.5751887798309326
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,4095,0.5609712123870849
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,4095,0.635862398147583
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,2047,2.29968318939209
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,2047,0.3101248025894165
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,1,0.15531359910964965
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,1,0.0851248025894165
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,1,0.07167999744415283
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,1,0.06382079720497132
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,1,0.06141759753227234
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,1,0.061212801933288576
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,1,0.06058400273323059
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,1,0.1635215997695923
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,1,0.09674080014228821
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,1,0.082777601480484
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,1,0.07495840191841126
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,1,0.0699504017829895
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,4095,4.143385696411133
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,1,0.07003200054168701
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,4095,4.125609588623047
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,1,0.06924160122871399
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,3,0.08681439757347106
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,4095,4.112582397460938
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,3,0.15658719539642335
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,3,0.07189440131187438
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,4095,4.1356464385986325
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,3,0.06099839806556702
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,3,0.06456800103187561
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,3,0.0614687979221344
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,3,0.061500799655914304
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,3,0.1670912027359009
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,3,0.08504800200462341
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,3,0.10220160484313964
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,3,0.07315199971199035
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,3,0.0720911979675293
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,3,0.07997599840164185
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,7,0.15437279939651488
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,7,0.07279199957847596
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,7,0.06548799872398377
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,7,0.061934399604797366
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,7,0.061103999614715576
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,7,0.061318397521972656
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,7,0.17462079524993895
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,7,0.10864160060882569
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,7,0.09294559955596923
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,7,0.08677120208740234
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,7,0.07930560111999511
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,7,0.0786191999912262
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,7,0.07771360278129577
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,15,0.15906399488449097
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,15,0.09009119868278503
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,15,0.07335360050201416
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,15,0.06241440176963806
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,15,0.061868798732757566
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,15,0.06169760227203369
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,4095,4.121446228027343
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,15,0.19368959665298463
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,4095,4.109966278076172
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,15,0.12875839471817016
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,15,0.10335040092468262
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,3,0.0719968020915985
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,7,0.0886896014213562
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,15,0.09191359877586365
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,31,0.16069600582122803
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,31,0.09182080030441284
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,31,0.06876159906387329
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,31,0.06438080072402955
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,31,0.06200640201568604
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,31,0.0612496018409729
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,31,0.22349119186401367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,31,0.15488640069961548
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,4095,4.509025573730469
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,31,0.13860319852828978
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,31,0.12801920175552367
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,31,0.13516000509262086
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,15,0.10700960159301758
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,63,0.10004639625549316
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,15,0.09794399738311768
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,63,0.08060799837112427
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,63,0.16844160556793214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,63,0.07367519736289978
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,15,0.09331520199775696
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,63,0.06922079920768738
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,63,0.06911680102348328
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,63,0.28138079643249514
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,31,0.07460319995880127
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,63,0.19135680198669433
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,63,0.18677120208740233
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,63,0.1874079942703247
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,63,0.18489279747009277
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,127,0.18773119449615477
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,15,0.06632480025291443
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,127,0.09852319955825806
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,127,0.09135680198669434
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,127,0.08526080250740051
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,127,0.08487200140953063
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,127,0.41558079719543456
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,31,0.12721920013427734
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,31,0.12797919511795045
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,127,0.346123194694519
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,127,0.32921600341796875
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,63,0.07075520157814026
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,127,0.32126240730285643
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,127,0.31652159690856935
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,127,0.31359679698944093
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,127,0.3130160093307495
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,255,0.23975839614868164
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,255,0.13193919658660888
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,255,0.12610399723052979
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,255,0.12044479846954345
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,255,0.11885279417037964
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,127,0.11787999868392944
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,255,0.6900432109832764
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,255,0.5992256164550781
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,127,0.08681759834289551
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,255,0.5834976196289062
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,255,0.5777023792266845
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,255,0.5736544132232666
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,255,0.5700575828552246
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,255,0.569820785522461
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,511,0.25271360874176024
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,511,0.40381760597229005
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,511,0.2221695899963379
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,511,0.20629758834838868
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,511,0.19857920408248902
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,63,0.20035998821258544
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,255,0.1474784016609192
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,255,0.1222607970237732
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,511,1.1487199783325195
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,511,1.2932319641113281
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,511,1.1004879951477051
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,511,1.0938976287841797
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,511,1.091198444366455
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,511,1.0891119956970214
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,511,0.19655200242996215
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,63,0.21735360622406005
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,511,0.19522240161895751
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,1023,0.3555232048034668
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,1023,0.3850863933563232
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,1023,0.6263599872589112
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,1023,0.3390064001083374
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,1023,0.32734079360961915
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,1023,0.3256911993026733
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,511,1.1163599967956543
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,1023,2.1690336227416993
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,1023,2.1182079315185547
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,1023,2.382991981506348
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,1023,2.1090431213378906
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,1023,2.1064687728881837
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,1023,0.3309904098510742
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,2047,0.6157919883728027
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,2047,0.6840367794036866
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,2047,0.6045807838439942
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,2047,0.5890768051147461
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,2047,1.0624591827392578
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,2047,0.5862912178039551
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,1023,2.1049472808837892
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,2047,0.5832320213317871
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,1023,2.1328815460205077
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,2047,4.13951530456543
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,2047,4.171633529663086
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,2047,4.139315032958985
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,2047,4.133665466308594
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,2047,4.156465530395508
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,2047,4.581699371337891
SGLang,0.5.9,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,2047,4.201153564453125
