framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,1,0.01234079971909523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,1,0.011832000315189361
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,1,0.01162080019712448
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,1,0.011635199934244157
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,1,0.011608000099658965
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,1,0.011563199758529662
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,1,0.011617600172758102
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,1,0.017948800325393678
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,1,0.01733119934797287
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,1,0.017316800355911256
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,1,0.017260800302028655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,1,0.017228800058364867
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,1,0.01706079989671707
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,1,0.017179200053215028
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,3,0.012251199781894683
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,3,0.0117807999253273
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,3,0.01154400035738945
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,3,0.011443199962377549
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,3,0.011342400312423706
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,3,0.011358399689197541
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,3,0.011561600118875503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,3,0.018055999279022218
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,3,0.017423999309539796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,3,0.017103999853134155
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,3,0.017105600237846373
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,3,0.01719360053539276
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,3,0.01722719967365265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,3,0.017187200486660004
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,7,0.012348800152540206
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,7,0.011881600320339202
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,7,0.011561600118875503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,7,0.011630400270223617
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,7,0.01130239963531494
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,7,0.01135680004954338
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,7,0.01156800016760826
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,7,0.017851200699806214
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,7,0.017507199943065644
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,7,0.01735839992761612
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,7,0.017150400578975676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,7,0.017164799571037292
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,7,0.01717119961977005
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,7,0.017156800627708434
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,15,0.012319999933242797
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,15,0.011846400052309036
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,15,0.011707200109958649
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,15,0.011727999895811081
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,15,0.011582399904727935
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,15,0.011615999788045884
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,15,0.011766400188207626
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,15,0.01794559955596924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,15,0.017496000230312347
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,15,0.017283199727535246
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,15,0.017350399494171144
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,15,0.01720159947872162
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,15,0.01717599928379059
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,15,0.017206400632858276
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,31,0.012404800206422806
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,31,0.011963199824094772
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,31,0.01170400008559227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,31,0.01170400008559227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,31,0.011665599793195725
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,31,0.011539199948310852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,31,0.011523199826478958
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,31,0.017951999604701997
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,31,0.01753759980201721
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,31,0.017423999309539796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,31,0.01722240000963211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,31,0.01724800020456314
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,31,0.01735360026359558
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,31,0.01721920073032379
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,63,0.012372799962759019
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,63,0.011923199892044068
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,63,0.011697600036859513
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,63,0.011713600158691407
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,63,0.011711999773979187
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,63,0.011699199676513672
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,63,0.011796800047159195
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,63,0.018161599338054658
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,63,0.017585599422454835
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,63,0.01752000004053116
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,63,0.017259199917316437
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,63,0.01725600063800812
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,63,0.017323200404644013
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,63,0.017476800084114074
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,127,0.014099200069904328
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,127,0.013672000169754029
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,127,0.0134320005774498
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,127,0.01327199935913086
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,127,0.01329759955406189
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,127,0.013240000605583191
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,127,0.013443200290203095
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,127,0.019969600439071655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,127,0.019351999461650848
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,127,0.019284799695014954
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,127,0.019201600551605226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,127,0.019156800210475923
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,127,0.01902880072593689
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,127,0.019113600254058838
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,255,0.017351999878883362
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,255,0.016892799735069276
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,255,0.01666879951953888
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,255,0.016492800414562227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,255,0.016473600268363954
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,255,0.016390399634838106
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,255,0.016366399824619293
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,255,0.022942399978637694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,255,0.02253119945526123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,255,0.022518399357795715
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,255,0.022336000204086305
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,255,0.02218399941921234
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,255,0.02227199971675873
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,255,0.022375999391078948
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,511,0.017985600233078002
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,511,0.016926400363445282
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,511,0.016113600134849547
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,511,0.015662400424480437
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,511,0.016935999691486358
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,511,0.01717280000448227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,511,0.01690559983253479
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,511,0.0238864004611969
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,511,0.022627200186252593
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,511,0.022147199511528014
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,511,0.021772800385951994
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,511,0.02269279956817627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,511,0.022939200699329376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,511,0.02284640073776245
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,1023,0.01813279986381531
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,1023,0.017110399901866913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,1023,0.016539199650287627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,1023,0.016232000291347505
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,1023,0.016979199647903443
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,1023,0.017148800194263458
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,1023,0.01698399931192398
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,1023,0.024303999543190003
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,1023,0.02340960055589676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,1023,0.022908799350261688
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,1023,0.022441600263118745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,1023,0.02343360036611557
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,1023,0.02340639978647232
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,1023,0.023263999819755556
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,2047,0.019044800102710722
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,2047,0.01788959950208664
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,2047,0.017214399576187134
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,2047,0.016972799599170686
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,2047,0.017161600291728973
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,2047,0.017185600101947786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,2047,0.01717119961977005
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,2047,0.027480000257492067
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,2047,0.02627840042114258
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,2047,0.025400000810623168
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,2047,0.025148800015449523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,2047,0.02545120120048523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,2047,0.02563680112361908
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,2047,0.02553279995918274
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,4095,0.0209184005856514
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,4095,0.019460800290107726
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,4095,0.018646399676799773
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,4095,0.018731200695037843
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,4095,0.019308799505233766
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,4095,0.019334399700164796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,4095,0.019305600225925444
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,4095,0.03213919997215271
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,4095,0.031062400341033934
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,4095,0.029820799827575684
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,4095,0.02975519895553589
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,4095,0.03054879903793335
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,4095,0.030772799253463747
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,4095,0.030475199222564697
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,8191,0.024139200150966645
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,8191,0.02206239998340607
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,8191,0.021564799547195434
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,8191,0.020899200439453126
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,8191,0.022392000257968902
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,8191,0.022782400250434875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,8191,0.022860799729824067
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,8191,0.04112800061702728
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,8191,0.03953279852867127
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,8191,0.038431999087333676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,8191,0.03813279867172241
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,8191,0.039603200554847715
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,8191,0.03984000086784363
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,8191,0.04002079963684082
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,16383,0.028059199452400208
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,16383,0.02526719868183136
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,16383,0.024489599466323852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,16383,0.024198399484157564
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,16383,0.024830399453639983
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,16383,0.0248879998922348
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,16383,0.02468000054359436
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,16383,0.06133440136909485
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,16383,0.05531039834022522
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,16383,0.05403519868850708
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,16383,0.05309600234031677
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,16383,0.053091198205947876
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,16383,0.053774398565292356
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,16383,0.05353119969367981
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,32767,0.03934879899024964
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,32767,0.03164480030536652
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,32767,0.02953760027885437
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,32767,0.02865760028362274
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,32767,0.02884480059146881
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,32767,0.028459200263023378
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,32767,0.02906239926815033
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,32767,0.09988800287246705
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,32767,0.0942031979560852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,32767,0.09327679872512817
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,32767,0.09288960099220275
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,32767,0.09236479997634887
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,32767,0.0924351990222931
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,32767,0.09220960140228271
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,65535,0.0479200005531311
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,65535,0.044835200905799864
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,65535,0.043119999766349795
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,65535,0.04187999963760376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,65535,0.04184800088405609
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,65535,0.04121440052986145
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,65535,0.04105280041694641
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,65535,0.16280000209808348
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,65535,0.16030240058898926
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,65535,0.1591231942176819
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,65535,0.15856800079345704
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,65535,0.1590656042098999
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,65535,0.15837279558181763
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,65535,0.15859199762344361
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,1,131071,0.0669871985912323
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,2,131071,0.06284000277519226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,4,131071,0.06001120209693909
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,8,131071,0.05912799835205078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,16,131071,0.05878559947013855
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,32,131071,0.05897279977798462
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1,1,64,131071,0.05832160115242004
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,1,131071,0.2912271976470947
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,2,131071,0.2881311893463135
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,4,131071,0.28697919845581055
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,8,131071,0.28683199882507326
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,16,131071,0.28611838817596436
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,32,131071,0.2859855890274048
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1,1,64,131071,0.28558080196380614
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,1,0.01218079999089241
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,1,0.011860799789428712
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,1,0.011753600090742111
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,1,0.011627200245857238
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,1,0.011559999734163284
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,1,0.01167680025100708
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,1,0.0115167997777462
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,1,0.01764480024576187
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,1,0.017561599612236023
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,1,0.01722719967365265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,1,0.017030400037765504
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,1,0.01695519983768463
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,1,0.017220799624919892
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,1,0.017177599668502807
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,3,0.012219200283288956
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,3,0.011726400256156922
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,3,0.011580800265073776
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,3,0.011406400054693223
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,3,0.011577600240707397
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,3,0.011699199676513672
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,3,0.011643200367689132
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,3,0.0176256000995636
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,3,0.017324799299240114
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,3,0.017364799976348877
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,3,0.017027199268341064
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,3,0.016832000017166136
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,3,0.017059199512004852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,3,0.017209599912166595
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,7,0.012454400211572647
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,7,0.011873599886894227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,7,0.011687999963760376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,7,0.011371199786663056
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,7,0.011470399796962738
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,7,0.011601600050926208
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,7,0.011689600348472596
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,7,0.017752000689506532
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,7,0.01729599982500076
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,7,0.0171424001455307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,7,0.01722719967365265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,7,0.017080000042915343
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,7,0.016971200704574585
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,7,0.0169295996427536
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,15,0.01255359947681427
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,15,0.0119439996778965
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,15,0.011990399658679962
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,15,0.011841599643230439
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,15,0.011814399808645248
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,15,0.011852800101041793
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,15,0.011691199988126755
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,15,0.01780800074338913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,15,0.017470400035381316
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,15,0.01748639941215515
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,15,0.01740960031747818
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,15,0.0171424001455307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,15,0.017110399901866913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,15,0.017182399332523347
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,31,0.012531200051307678
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,31,0.011964800208806992
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,31,0.011812800168991089
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,31,0.011547199636697768
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,31,0.011649599671363831
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,31,0.011767999827861786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,31,0.011776000261306763
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,31,0.01797119975090027
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,31,0.017422400414943695
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,31,0.017110399901866913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,31,0.017185600101947786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,31,0.017195199429988862
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,31,0.017113600671291352
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,31,0.017164799571037292
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,63,0.012579199671745301
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,63,0.012062399834394454
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,63,0.011811199784278869
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,63,0.01159520000219345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,63,0.011593600362539291
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,63,0.011695999652147293
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,63,0.01162559986114502
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,63,0.018105599284172057
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,63,0.017774400115013123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,63,0.01746239960193634
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,63,0.017156800627708434
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,63,0.017260800302028655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,63,0.017403200268745422
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,63,0.017103999853134155
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,127,0.014295999705791474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,127,0.013798399269580841
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,127,0.013670399785041809
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,127,0.013548800349235534
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,127,0.013497599959373474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,127,0.01340479999780655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,127,0.013356800377368926
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,127,0.019627200067043306
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,127,0.01919520050287247
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,127,0.01915840059518814
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,127,0.0189983993768692
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,127,0.018918399512767792
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,127,0.01881919950246811
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,127,0.01900479942560196
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,255,0.017472000420093538
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,255,0.016926400363445282
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,255,0.016811199486255646
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,255,0.016606399416923524
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,255,0.016451199352741242
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,255,0.016590400040149687
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,255,0.016550399363040924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,255,0.023028799891471864
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,255,0.022678400576114654
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,255,0.022416000068187714
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,255,0.022251200675964356
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,255,0.022305600345134735
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,255,0.022249600291252135
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,255,0.022150400280952453
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,511,0.017960000038146972
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,511,0.01682240068912506
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,511,0.01626719981431961
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,511,0.015902400016784668
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,511,0.016950400173664094
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,511,0.01706079989671707
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,511,0.017059199512004852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,511,0.024271999299526215
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,511,0.022888000309467315
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,511,0.022227199375629426
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,511,0.022009600698947907
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,511,0.023099200427532197
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,511,0.023227199912071228
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,511,0.023081600666046143
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,1023,0.018739199638366698
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,1023,0.017236800491809846
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,1023,0.016791999340057373
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,1023,0.016278399527072905
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,1023,0.017150400578975676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,1023,0.01738879978656769
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,1023,0.017387199401855468
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,1023,0.026875200867652892
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,1023,0.025300800800323486
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,1023,0.024689599871635437
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,1023,0.024299199879169463
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,1023,0.02518239915370941
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,1023,0.025283199548721314
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,1023,0.02518880069255829
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,2047,0.01948480010032654
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,2047,0.01786080002784729
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,2047,0.017427200078964235
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,2047,0.0171424001455307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,2047,0.017470400035381316
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,2047,0.01764000058174133
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,2047,0.017492799460887908
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,2047,0.030327999591827394
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,2047,0.028854399919509888
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,2047,0.028297600150108338
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,2047,0.028038400411605834
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,2047,0.028118398785591126
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,2047,0.02841919958591461
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,2047,0.028188800811767577
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,4095,0.021382400393486024
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,4095,0.01958560049533844
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,4095,0.019065600633621217
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,4095,0.01873279958963394
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,4095,0.01937599927186966
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,4095,0.019625599682331085
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,4095,0.019364799559116363
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,4095,0.0386927992105484
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,4095,0.03665600121021271
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,4095,0.03604960143566131
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,4095,0.03553119897842407
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,4095,0.03638559877872467
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,4095,0.03634240031242371
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,4095,0.03606880009174347
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,8191,0.026263999938964843
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,8191,0.02250719964504242
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,8191,0.021937599778175353
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,8191,0.021478399634361267
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,8191,0.024063999950885772
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,8191,0.024486400187015533
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,8191,0.024480000138282776
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,8191,0.05983039736747742
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,8191,0.05235840082168579
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,8191,0.05091519951820374
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,8191,0.05021920204162598
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,8191,0.05268480181694031
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,8191,0.05320960283279419
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,8191,0.053035199642181396
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,16383,0.03701280057430267
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,16383,0.02877120077610016
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,16383,0.026096001267433167
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,16383,0.025684800744056702
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,16383,0.027395200729370118
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,16383,0.0278656005859375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,16383,0.02812960147857666
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,16383,0.09792799949645996
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,16383,0.09175360202789307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,16383,0.08962560296058655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,16383,0.08896960020065307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,16383,0.09115679860115052
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,16383,0.09171839952468872
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,16383,0.09192000031471252
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,32767,0.050012797117233276
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,32767,0.04049600064754486
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,32767,0.03868640065193176
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,32767,0.037699198722839354
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,32767,0.03947199881076813
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,32767,0.03970080018043518
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,32767,0.039684799313545224
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,32767,0.16184799671173095
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,32767,0.15568000078201294
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,32767,0.15484479665756226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,32767,0.15421119928359986
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,32767,0.15601919889450072
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,32767,0.15646239519119262
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,32767,0.15640159845352172
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,65535,0.0655232012271881
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,65535,0.05854719877243042
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,65535,0.05649600028991699
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,65535,0.05522400140762329
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,65535,0.057257598638534545
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,65535,0.057548797130584715
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,65535,0.057201600074768065
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,65535,0.2884608030319214
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,65535,0.2846384048461914
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,65535,0.28371200561523435
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,65535,0.28297441005706786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,65535,0.2847615957260132
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,65535,0.2848527908325195
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,65535,0.28514559268951417
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,1,131071,0.09987679719924927
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,2,131071,0.0941424012184143
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,4,131071,0.09208480119705201
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,8,131071,0.09046080112457275
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,16,131071,0.09205279946327209
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,32,131071,0.09281920194625855
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,2,1,64,131071,0.0922544002532959
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,1,131071,0.5397136211395264
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,2,131071,0.536678409576416
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,4,131071,0.5352528095245361
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,8,131071,0.5349520206451416
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,16,131071,0.5358880043029786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,32,131071,0.5366159915924072
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,2,1,64,131071,0.5366864204406738
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,1,0.01225920021533966
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,1,0.01178399994969368
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,1,0.011598400026559829
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,1,0.011575999855995178
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,1,0.01141119971871376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,1,0.011659199744462967
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,1,0.011566399782896041
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,1,0.01802079975605011
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,1,0.01740960031747818
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,1,0.017220799624919892
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,1,0.017150400578975676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,1,0.017259199917316437
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,1,0.017099200189113616
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,1,0.01722400039434433
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,3,0.012464000284671784
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,3,0.011987199634313583
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,3,0.011774399876594543
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,3,0.011572799831628799
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,3,0.011559999734163284
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,3,0.011598400026559829
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,3,0.011507199704647064
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,3,0.01793919950723648
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,3,0.01744000017642975
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,3,0.017382399737834932
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,3,0.017209599912166595
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,3,0.017319999635219574
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,3,0.017107200622558594
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,3,0.0171984001994133
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,7,0.012244799733161926
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,7,0.01199359968304634
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,7,0.011718399822711945
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,7,0.011742399632930755
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,7,0.011667200177907944
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,7,0.011699199676513672
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,7,0.011670400202274323
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,7,0.01794240027666092
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,7,0.017420800030231477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,7,0.017315199971199034
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,7,0.01738400012254715
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,7,0.01732800006866455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,7,0.017047999799251555
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,7,0.01720000058412552
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,15,0.012532800436019897
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,15,0.01213119998574257
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,15,0.011803200095891952
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,15,0.011772800236940384
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,15,0.01167839989066124
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,15,0.011603199690580369
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,15,0.011604800075292587
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,15,0.01799360066652298
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,15,0.017529599368572235
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,15,0.017360000312328337
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,15,0.017286400496959686
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,15,0.01743520051240921
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,15,0.01727679967880249
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,15,0.01716960072517395
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,31,0.012319999933242797
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,31,0.012051200121641159
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,31,0.011767999827861786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,31,0.011812800168991089
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,31,0.011787199974060058
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,31,0.01170239970088005
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,31,0.01180799975991249
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,31,0.018057599663734436
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,31,0.017632000148296356
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,31,0.017423999309539796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,31,0.017211200296878816
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,31,0.017188799381256104
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,31,0.017417599260807038
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,31,0.017446400225162507
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,63,0.012478400021791458
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,63,0.012033600360155106
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,63,0.011817599833011627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,63,0.011788800358772278
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,63,0.011604800075292587
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,63,0.011628799885511399
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,63,0.01180960014462471
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,63,0.018115200102329254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,63,0.017668800055980684
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,63,0.017603200674057008
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,63,0.01741439998149872
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,63,0.017411200702190398
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,63,0.017347200214862822
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,63,0.01735839992761612
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,127,0.014284799993038177
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,127,0.01395999938249588
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,127,0.013633599877357483
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,127,0.013606399297714233
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,127,0.013457599282264709
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,127,0.013371199369430542
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,127,0.013484799861907959
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,127,0.020214399695396422
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,127,0.01951040029525757
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,127,0.019288000464439393
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,127,0.01916320025920868
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,127,0.019355200231075287
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,127,0.01929280012845993
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,127,0.019054399430751802
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,255,0.017238399386405943
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,255,0.016942399740219116
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,255,0.01676799952983856
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,255,0.016771200299263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,255,0.01658080071210861
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,255,0.016468800604343414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,255,0.01653279960155487
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,255,0.0234592005610466
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,255,0.023148800432682037
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,255,0.022731199860572815
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,255,0.02261279970407486
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,255,0.022606399655342103
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,255,0.022745600342750548
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,255,0.022801600396633148
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,511,0.018641600012779237
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,511,0.017132799327373504
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,511,0.016828800737857818
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,511,0.01631679981946945
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,511,0.01695840060710907
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,511,0.017371200025081635
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,511,0.017347200214862822
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,511,0.027001601457595826
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,511,0.025628799200057985
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,511,0.024619199335575104
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,511,0.024270400404930115
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,511,0.025228801369667053
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,511,0.025593599677085875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,511,0.025548800826072693
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,1023,0.019124799966812135
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,1023,0.017692799866199493
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,1023,0.016990399360656737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,1023,0.016548800468444824
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,1023,0.017217600345611574
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,1023,0.01746399998664856
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,1023,0.017472000420093538
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,1023,0.030212798714637758
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,1023,0.028887999057769776
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,1023,0.02773439884185791
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,1023,0.02748799920082092
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,1023,0.0283488005399704
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,1023,0.028620800375938414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,1023,0.028417599201202393
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,2047,0.021267199516296388
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,2047,0.019251200556755065
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,2047,0.017924800515174866
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,2047,0.018016000092029572
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,2047,0.018214400112628936
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,2047,0.018083199858665466
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,2047,0.01780160069465637
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,2047,0.03826079964637756
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,2047,0.03579359948635101
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,2047,0.03490079939365387
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,2047,0.03462719917297363
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,2047,0.03497599959373474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,2047,0.034995201230049136
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,2047,0.034852799773216245
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,4095,0.025316798686981203
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,4095,0.022152000665664674
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,4095,0.020524799823760986
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,4095,0.02011999934911728
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,4095,0.02125760018825531
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,4095,0.02155359983444214
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,4095,0.021726399660110474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,4095,0.05944640040397644
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,4095,0.051611202955245974
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,4095,0.04946399927139282
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,4095,0.048879998922348025
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,4095,0.049835199117660524
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,4095,0.05009120106697083
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,4095,0.05009120106697083
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,8191,0.034825599193573
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,8191,0.026769599318504332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,8191,0.02476159930229187
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,8191,0.024001599848270418
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,8191,0.027673599123954774
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,8191,0.028313601016998292
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,8191,0.028302401304244995
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,8191,0.09632319808006287
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,8191,0.08995199799537659
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,8191,0.0883247971534729
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,8191,0.0877120018005371
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,8191,0.0919152021408081
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,8191,0.09181920289993287
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,8191,0.09204639792442322
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,16383,0.048132801055908205
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,16383,0.03958399891853333
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,16383,0.036792001128196715
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,16383,0.03589600026607513
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,16383,0.039192000031471254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,16383,0.0399152010679245
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,16383,0.04023360013961792
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,16383,0.1614735960960388
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,16383,0.15402239561080933
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,16383,0.1535312056541443
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,16383,0.15286879539489745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,16383,0.15598080158233643
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,16383,0.15702400207519532
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,16383,0.15644799470901488
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,32767,0.063508802652359
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,32767,0.05729280114173889
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,32767,0.05514879822731018
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,32767,0.05376480221748352
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,32767,0.05716000199317932
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,32767,0.05783360004425049
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,32767,0.05743839740753174
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,32767,0.2876816034317017
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,32767,0.283351993560791
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,32767,0.28231680393218994
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,32767,0.28119359016418455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,32767,0.2850944042205811
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,32767,0.28525280952453613
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,32767,0.2854223966598511
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,65535,0.09818080067634583
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,65535,0.09260640144348145
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,65535,0.08992959856987
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,65535,0.08898400068283081
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,65535,0.09281600117683411
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,65535,0.09284639954566956
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,65535,0.09273920059204102
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,65535,0.5385968208312988
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,65535,0.5347663879394531
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,65535,0.5335599899291992
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,65535,0.5326223850250245
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,65535,0.5357471942901612
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,65535,0.5367616176605224
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,65535,0.5363183975219726
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,1,131071,0.1672927975654602
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,2,131071,0.16053919792175292
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,4,131071,0.1598031997680664
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,8,131071,0.15732799768447875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,16,131071,0.16039199829101564
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,32,131071,0.16125439405441283
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,4,1,64,131071,0.16117759943008422
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,1,131071,1.0401616096496582
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,2,131071,1.0349167823791503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,4,131071,1.0343839645385742
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,8,131071,1.0324543952941894
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,16,131071,1.037235164642334
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,32,131071,1.0365216255187988
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,4,1,64,131071,1.0371487617492676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,1,0.012539200484752655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,1,0.01186240017414093
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,1,0.01188800036907196
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,1,0.011519999802112579
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,1,0.01191840022802353
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,1,0.011713600158691407
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,1,0.011798399686813354
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,1,0.018062399327754976
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,1,0.017899200320243835
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,1,0.0176256000995636
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,1,0.017560000717639922
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,1,0.01736319959163666
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,1,0.017521600425243377
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,1,0.017548799514770508
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,3,0.012759999930858612
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,3,0.012083200365304947
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,3,0.011912000179290772
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,3,0.011596799641847611
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,3,0.011919999867677689
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,3,0.011793600022792816
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,3,0.011919999867677689
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,3,0.0181551992893219
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,3,0.01770080029964447
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,3,0.017539200186729432
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,3,0.017612800002098083
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,3,0.017510400712490083
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,3,0.01738879978656769
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,3,0.017263999581336974
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,7,0.012779200077056884
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,7,0.012062399834394454
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,7,0.012076800316572189
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,7,0.011745599657297134
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,7,0.011868800222873687
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,7,0.01159999966621399
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,7,0.011740799993276596
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,7,0.01804800033569336
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,7,0.01777759939432144
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,7,0.01772480010986328
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,7,0.017423999309539796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,7,0.01733759939670563
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,7,0.017526400089263917
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,7,0.017577600479125977
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,15,0.01263200044631958
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,15,0.011958400160074234
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,15,0.012134400010108948
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,15,0.011798399686813354
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,15,0.012078399956226348
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,15,0.011847999691963196
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,15,0.011984000355005265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,15,0.018214400112628936
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,15,0.01772480010986328
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,15,0.017636799812316896
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,15,0.017667199671268462
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,15,0.017657600343227386
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,15,0.017291200160980225
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,15,0.017604799568653108
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,31,0.012547199428081513
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,31,0.01199679970741272
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,31,0.011812800168991089
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,31,0.01170400008559227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,31,0.01175519973039627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,31,0.011718399822711945
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,31,0.011910399794578553
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,31,0.01871040016412735
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,31,0.018001599609851836
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,31,0.01781280040740967
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,31,0.017560000717639922
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,31,0.01759359985589981
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,31,0.017671999335289002
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,31,0.017772799730300902
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,63,0.012849600613117218
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,63,0.012204799801111221
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,63,0.012267199903726577
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,63,0.011924800276756287
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,63,0.011875200271606445
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,63,0.011687999963760376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,63,0.011684799939393998
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,63,0.01846559941768646
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,63,0.018241600692272188
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,63,0.01820800006389618
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,63,0.017899200320243835
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,63,0.017712000012397765
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,63,0.01793919950723648
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,63,0.017961600422859193
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,127,0.014452800154685974
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,127,0.013928000628948212
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,127,0.013782399892807006
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,127,0.01342879980802536
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,127,0.013603200018405915
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,127,0.01348000019788742
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,127,0.01358560025691986
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,127,0.020737600326538087
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,127,0.020284800231456755
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,127,0.02022079974412918
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,127,0.019713599979877473
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,127,0.019993600249290467
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,127,0.019734400510787963
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,127,0.01990559995174408
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,255,0.017561599612236023
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,255,0.017046399414539337
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,255,0.016926400363445282
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,255,0.016729600727558136
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,255,0.016760000586509706
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,255,0.016652800142765045
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,255,0.016571199893951415
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,255,0.02558720111846924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,255,0.025068798661231996
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,255,0.025060799717903138
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,255,0.02476159930229187
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,255,0.024953599274158477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,255,0.024828800559043886
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,255,0.024804799258708952
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,511,0.01929440051317215
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,511,0.017521600425243377
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,511,0.017033599317073822
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,511,0.016436800360679626
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,511,0.017547200620174407
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,511,0.017694400250911714
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,511,0.017662400007247926
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,511,0.030451199412345885
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,511,0.028784000873565675
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,511,0.027939200401306152
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,511,0.027531200647354127
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,511,0.028784000873565675
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,511,0.028830400109291075
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,511,0.0287663996219635
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,1023,0.021222400665283202
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,1023,0.019088000059127808
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,1023,0.01759839951992035
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,1023,0.017254400253295898
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,1023,0.01823839992284775
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,1023,0.018217599391937254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,1023,0.018329599499702455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,1023,0.03868800103664398
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,1023,0.03570559918880463
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,1023,0.03481760025024414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,1023,0.03434880077838898
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,1023,0.03505440056324005
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,1023,0.03537440001964569
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,1023,0.035492798686027525
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,2047,0.025620800256729127
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,2047,0.021881599724292756
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,2047,0.02008800059556961
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,2047,0.019628800451755524
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,2047,0.020110400021076204
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,2047,0.020132799446582795
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,2047,0.01998399943113327
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,2047,0.05902079939842224
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,2047,0.051815998554229734
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,2047,0.049534401297569274
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,2047,0.048332801461219786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,2047,0.04878720045089722
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,2047,0.04869279861450195
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,2047,0.048742398619651794
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,4095,0.03366079926490784
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,4095,0.027582401037216188
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,4095,0.024291199445724488
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,4095,0.023398399353027344
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,4095,0.024806399643421174
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,4095,0.025190401077270507
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,4095,0.025102400779724122
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,4095,0.09411680102348327
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,4095,0.0901199996471405
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,4095,0.08803359866142273
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,4095,0.08729919791221619
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,4095,0.08859999775886536
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,4095,0.08897439837455749
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,4095,0.08914719820022583
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,8191,0.04525279998779297
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,8191,0.039689600467681885
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,8191,0.03630560040473938
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,8191,0.035071998834609985
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,8191,0.0391184002161026
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,8191,0.039854401350021364
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,8191,0.04000479876995087
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,8191,0.15983999967575074
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,8191,0.15451680421829223
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,8191,0.1525007963180542
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,8191,0.15209120512008667
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,8191,0.15681439638137817
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,8191,0.15736960172653197
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,8191,0.15720959901809692
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,16383,0.06411200165748596
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,16383,0.05693759918212891
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,16383,0.054422402381896974
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,16383,0.05314720273017883
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,16383,0.056999999284744265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,16383,0.05781919956207275
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,16383,0.05781279802322388
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,16383,0.2877455949783325
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,16383,0.28273439407348633
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,16383,0.28105599880218507
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,16383,0.2802448034286499
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,16383,0.28479681015014646
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,16383,0.28591361045837405
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,16383,0.28562400341033933
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,32767,0.09947199821472168
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,32767,0.09249759912490844
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,32767,0.08933119773864746
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,32767,0.08790079951286316
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,32767,0.09311360120773315
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,32767,0.0935263991355896
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,32767,0.09322400093078613
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,32767,0.5393968105316163
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,32767,0.534819221496582
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,32767,0.5326496124267578
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,32767,0.5319183826446533
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,32767,0.5360943794250488
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,32767,0.5370031833648682
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,32767,0.5371312141418457
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,65535,0.1667215943336487
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,65535,0.1600800037384033
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,65535,0.15803200006484985
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,65535,0.1571616053581238
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,65535,0.16024320125579833
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,65535,0.16167360544204712
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,65535,0.16056640148162843
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,65535,1.0383616447448731
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,65535,1.034604835510254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,65535,1.0329615592956543
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,65535,1.0325728416442872
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,65535,1.0371295928955078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,65535,1.036518383026123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,65535,1.0362208366394043
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,1,131071,0.30760319232940675
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,2,131071,0.30038559436798096
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,4,131071,0.29513280391693114
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,8,131071,0.29472639560699465
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,16,131071,0.29747838973999025
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,32,131071,0.2975600004196167
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,8,1,64,131071,0.29823520183563235
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,1,131071,2.0395551681518556
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,2,131071,2.0366512298583985
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,4,131071,2.0344831466674806
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,8,131071,2.0312063217163088
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,16,131071,2.0367504119873048
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,32,131071,2.0365760803222654
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,8,1,64,131071,2.0369359970092775
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,1,0.012651200592517852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,1,0.012443199753761292
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,1,0.011847999691963196
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,1,0.01202239990234375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,1,0.011747200042009354
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,1,0.0119439996778965
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,1,0.011764799803495407
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,1,0.01812320053577423
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,1,0.017985600233078002
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,1,0.01781280040740967
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,1,0.017448000609874725
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,1,0.0176816001534462
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,1,0.017396800220012665
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,1,0.01738879978656769
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,3,0.012796799838542938
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,3,0.012441600114107132
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,3,0.01196959987282753
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,3,0.012167999893426895
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,3,0.012049599736928939
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,3,0.011953599750995636
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,3,0.011771199852228164
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,3,0.018087999522686006
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,3,0.017852799594402315
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,3,0.017844800651073457
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,3,0.017550399899482726
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,3,0.017601600289344786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,3,0.017420800030231477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,3,0.017528000473976135
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,7,0.013067199289798737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,7,0.01242400035262108
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,7,0.01199359968304634
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,7,0.011958400160074234
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,7,0.011761599779129028
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,7,0.011932799965143204
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,7,0.01178240031003952
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,7,0.018481600284576415
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,7,0.018027199804782866
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,7,0.01757120043039322
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,7,0.017726400494575502
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,7,0.017828799784183502
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,7,0.01775840073823929
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,7,0.017561599612236023
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,15,0.01313759982585907
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,15,0.012544000148773193
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,15,0.01207199990749359
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,15,0.012038400024175644
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,15,0.011988800019025803
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,15,0.01196800023317337
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,15,0.011894399672746659
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,15,0.01876640021800995
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,15,0.018326400220394133
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,15,0.01754560023546219
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,15,0.01743199974298477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,15,0.01751679927110672
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,15,0.017555199563503265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,15,0.01763039976358414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,31,0.013310399651527405
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,31,0.01249919980764389
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,31,0.01199679970741272
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,31,0.012188799679279327
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,31,0.011979199945926666
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,31,0.012001600116491318
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,31,0.011984000355005265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,31,0.019009600579738616
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,31,0.017998400330543517
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,31,0.018116800487041472
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,31,0.017766399681568144
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,31,0.01788959950208664
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,31,0.017638400197029114
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,31,0.01807039976119995
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,63,0.013756799697875976
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,63,0.01244800016283989
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,63,0.012147200107574464
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,63,0.012031999975442886
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,63,0.011952000111341477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,63,0.012076800316572189
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,63,0.011830399930477142
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,63,0.019739200174808503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,63,0.018848000466823576
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,63,0.018177600204944612
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,63,0.018185600638389587
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,63,0.018115200102329254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,63,0.018372799456119537
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,63,0.018294399976730345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,127,0.014711999893188476
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,127,0.014107200503349304
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,127,0.013798399269580841
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,127,0.01390880048274994
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,127,0.013766400516033173
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,127,0.013990400731563568
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,127,0.013571199774742127
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,127,0.022465600073337554
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,127,0.022065599262714387
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,127,0.021905599534511565
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,127,0.02186879962682724
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,127,0.02156960070133209
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,127,0.021515199542045595
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,127,0.021508799493312837
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,255,0.01849119961261749
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,255,0.017340800166130065
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,255,0.016847999393939973
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,255,0.016993600130081176
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,255,0.0169855996966362
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,255,0.017099200189113616
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,255,0.016976000368595125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,255,0.028572800755500793
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,255,0.027888000011444092
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,255,0.02770560085773468
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,255,0.027747198939323425
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,255,0.02768160104751587
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,255,0.027745598554611207
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,255,0.027622398734092713
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,511,0.02154400050640106
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,511,0.018913599848747253
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,511,0.017638400197029114
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,511,0.016892799735069276
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,511,0.01775359958410263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,511,0.01846559941768646
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,511,0.01836639940738678
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,511,0.03844479918479919
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,511,0.03607040047645569
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,511,0.03434560000896454
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,511,0.03406560122966766
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,511,0.0350735992193222
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,511,0.03514719903469086
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,511,0.03497920036315918
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,1023,0.02473919987678528
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,1023,0.022121599316596983
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,1023,0.020316800475120543
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,1023,0.019312000274658202
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,1023,0.02036159932613373
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,1023,0.021267199516296388
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,1023,0.02115200012922287
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,1023,0.05835199952125549
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,1023,0.05209280252456665
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,1023,0.04949440062046051
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,1023,0.04865919947624207
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,1023,0.04901919960975647
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,1023,0.04960319995880127
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,1023,0.04972319900989532
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,2047,0.0321152001619339
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,2047,0.02727839946746826
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,2047,0.024828800559043886
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,2047,0.02314720004796982
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,2047,0.023790399730205535
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,2047,0.0247871994972229
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,2047,0.024700799584388734
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,2047,0.09255679845809936
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,2047,0.08913599848747253
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,2047,0.08731520175933838
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,2047,0.08627359867095948
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,2047,0.08702239990234376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,2047,0.08733919858932496
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,2047,0.08754879832267762
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,4095,0.043584001064300534
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,4095,0.038815999031066896
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,4095,0.0364192008972168
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,4095,0.034657600522041324
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,4095,0.0367792010307312
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,4095,0.03701280057430267
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,4095,0.03749119937419891
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,4095,0.15716960430145263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,4095,0.1534432053565979
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,4095,0.15156639814376832
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,4095,0.1507439970970154
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,4095,0.15219199657440186
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,4095,0.1528447985649109
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,4095,0.15274879932403565
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,8191,0.06214720010757446
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,8191,0.056760001182556155
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,8191,0.053643202781677245
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,8191,0.05223360061645508
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,8191,0.05676959753036499
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,8191,0.05791040062904358
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,8191,0.05780640244483948
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,8191,0.28586881160736083
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,8191,0.2818063974380493
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,8191,0.28015201091766356
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,8191,0.2791824102401733
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,8191,0.28359360694885255
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,8191,0.28467841148376466
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,8191,0.284169602394104
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,16383,0.09741119742393493
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,16383,0.09130560159683228
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,16383,0.08902879953384399
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,16383,0.08716319799423218
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,16383,0.09249759912490844
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,16383,0.09287999868392945
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,16383,0.09342079758644103
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,16383,0.5381184101104737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,16383,0.5337296009063721
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,16383,0.5322847843170166
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,16383,0.5320367813110352
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,16383,0.5364655971527099
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,16383,0.5371871948242187
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,16383,0.5373871803283692
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,32767,0.1658992052078247
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,32767,0.1603808045387268
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,32767,0.15633920431137086
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,32767,0.15529760122299194
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,32767,0.15953119993209838
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,32767,0.16100640296936036
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,32767,0.16069120168685913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,32767,1.0381551742553712
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,32767,1.0337072372436524
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,32767,1.0327823638916016
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,32767,1.0326767921447755
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,32767,1.0379152297973633
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,32767,1.037662410736084
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,32767,1.0371952056884766
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,65535,0.3018671989440918
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,65535,0.29217119216918946
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,65535,0.28978240489959717
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,65535,0.28784639835357667
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,65535,0.29317440986633303
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,65535,0.2926543951034546
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,65535,0.2922224044799805
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,65535,2.0377599716186525
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,65535,2.032316780090332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,65535,2.031398391723633
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,65535,2.0298112869262694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,65535,2.0342960357666016
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,65535,2.0348831176757813
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,65535,2.035004806518555
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,1,131071,0.5804575920104981
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,2,131071,0.5651696205139161
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,4,131071,0.5581200122833252
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,8,131071,0.5578271865844726
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,16,131071,0.560753583908081
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,32,131071,0.5639103889465332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,16,1,64,131071,0.5616991996765137
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,1,131071,4.055755233764648
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,2,131071,4.047102355957032
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,4,131071,4.030624008178711
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,8,131071,4.027427291870117
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,16,131071,4.0317840576171875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,32,131071,4.032444763183594
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,16,1,64,131071,4.03230094909668
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,1,0.013729600608348847
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,1,0.01297920048236847
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,1,0.013195200264453888
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,1,0.012654399871826172
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,1,0.012689599394798278
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,1,0.012667199969291687
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,1,0.01268800050020218
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,1,0.019280000030994414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,1,0.01860959976911545
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,1,0.018459199368953703
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,1,0.01820479929447174
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,1,0.018016000092029572
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,1,0.018212799727916718
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,1,0.0181536003947258
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,3,0.013607999682426453
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,3,0.01297760009765625
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,3,0.012615999579429627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,3,0.012455999851226807
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,3,0.012507200241088867
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,3,0.01242400035262108
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,3,0.01252799928188324
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,3,0.01934719979763031
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,3,0.01870879977941513
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,3,0.01839359998703003
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,3,0.01801760047674179
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,3,0.018294399976730345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,3,0.01825920045375824
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,3,0.01825280040502548
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,7,0.013876800239086152
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,7,0.01313599944114685
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,7,0.012945599853992462
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,7,0.012680000066757202
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,7,0.012783999741077422
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,7,0.012537600100040435
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,7,0.012591999769210816
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,7,0.019308799505233766
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,7,0.01873600035905838
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,7,0.018622399866580965
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,7,0.018409599363803864
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,7,0.018303999304771425
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,7,0.018488000333309173
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,7,0.018199999630451203
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,15,0.013687999546527862
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,15,0.013036799430847169
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,15,0.013007999956607818
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,15,0.012750400602817536
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,15,0.012641599774360657
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,15,0.013291199505329133
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,15,0.012915199995040894
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,15,0.01958079934120178
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,15,0.01865759938955307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,15,0.01844319999217987
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,15,0.01866399943828583
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,15,0.01826079934835434
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,15,0.018456000089645385
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,15,0.01855040043592453
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,31,0.014076800644397735
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,31,0.013107199966907502
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,31,0.012857599556446076
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,31,0.012587200105190276
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,31,0.012723200023174286
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,31,0.012601600587368011
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,31,0.012878400087356568
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,31,0.02029920071363449
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,31,0.019315199553966524
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,31,0.018873600661754607
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,31,0.018675200641155243
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,31,0.018854400515556334
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,31,0.018854400515556334
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,31,0.01923840045928955
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,63,0.014388799667358398
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,63,0.013414399325847625
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,63,0.013254399597644805
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,63,0.012948800623416901
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,63,0.013046400249004364
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,63,0.013158400356769562
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,63,0.01326880007982254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,63,0.022673599421977997
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,63,0.021220800280570985
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,63,0.021244800090789794
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,63,0.021033599972724915
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,63,0.02093600034713745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,63,0.02093919962644577
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,63,0.021211199462413788
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,127,0.016094399988651274
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,127,0.015300799906253815
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,127,0.015015999972820281
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,127,0.014867199957370758
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,127,0.01499519944190979
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,127,0.014619199931621552
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,127,0.014705599844455719
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,127,0.02696320116519928
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,127,0.02587519884109497
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,127,0.025676798820495606
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,127,0.02555519938468933
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,127,0.025598400831222536
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,127,0.025302401185035704
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,127,0.025352001190185547
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,255,0.019486400485038757
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,255,0.018268799781799315
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,255,0.018187199532985688
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,255,0.017987200617790224
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,255,0.018217599391937254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,255,0.017870399355888366
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,255,0.017795200645923614
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,255,0.035848000645637514
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,255,0.034780800342559814
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,255,0.03441919982433319
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,255,0.034176000952720643
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,255,0.03454720079898834
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,255,0.03451200127601624
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,255,0.0344976007938385
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,511,0.026390400528907777
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,511,0.021524800360202788
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,511,0.020692799985408784
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,511,0.019491200149059296
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,511,0.020828799903392793
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,511,0.021457600593566894
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,511,0.021595199406147004
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,511,0.06067360043525696
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,511,0.053067201375961305
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,511,0.04971520006656647
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,511,0.04820800125598908
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,511,0.049737599492073056
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,511,0.050569599866867064
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,511,0.05048480033874512
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,1023,0.03461439907550812
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,1023,0.026804798841476442
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,1023,0.024480000138282776
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,1023,0.02332639992237091
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,1023,0.026009601354599
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,1023,0.027580800652503967
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,1023,0.027961599826812743
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,1023,0.09373120069503785
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,1023,0.08796960115432739
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,1023,0.08661119937896729
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,1023,0.08575999736785889
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,1023,0.08772000074386596
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,1023,0.08861600160598755
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,1023,0.08954399824142456
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,2047,0.045470398664474485
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,2047,0.03856480121612549
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,2047,0.03670240044593811
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,2047,0.03511520028114319
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,2047,0.03614720106124878
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,2047,0.03696160018444061
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,2047,0.03698720037937164
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,2047,0.15917279720306396
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,2047,0.15358400344848633
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,2047,0.15207200050354003
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,2047,0.1501744031906128
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,2047,0.15172799825668334
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,2047,0.15290240049362183
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,2047,0.15251840353012086
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,4095,0.06408320069313049
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,4095,0.05705919861793518
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,4095,0.054313600063323975
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,4095,0.05261279940605164
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,4095,0.055099201202392575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,4095,0.05716480016708374
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,4095,0.05744640231132507
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,4095,0.2879472017288208
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,4095,0.2824640035629272
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,4095,0.28050880432128905
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,4095,0.2793744087219238
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,4095,0.282041597366333
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,4095,0.28363358974456787
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,4095,0.2834111928939819
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,8191,0.09965919852256774
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,8191,0.09256319999694824
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,8191,0.089273601770401
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,8191,0.08753439784049988
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,8191,0.09409919977188111
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,8191,0.09793440103530884
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,8191,0.09709439873695373
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,8191,0.5400335788726807
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,8191,0.5359807968139648
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,8191,0.5332096099853516
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,8191,0.5317984104156495
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,8191,0.537224006652832
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,8191,0.5403520107269287
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,8191,0.5411871910095215
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,16383,0.1684720039367676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,16383,0.16094720363616943
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,16383,0.15802719593048095
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,16383,0.1557487964630127
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,16383,0.16101919412612914
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,16383,0.16357760429382323
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,16383,0.16323519945144654
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,16383,1.0393263816833496
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,16383,1.0349040031433105
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,16383,1.03230562210083
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,16383,1.0315823554992676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,16383,1.0366991996765136
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,16383,1.038751983642578
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,16383,1.038148784637451
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,32767,0.30417919158935547
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,32767,0.29470880031585694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,32767,0.2916208028793335
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,32767,0.2909359931945801
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,32767,0.29318881034851074
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,32767,0.2953775882720947
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,32767,0.29478719234466555
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,32767,2.037980842590332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,32767,2.0353872299194338
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,32767,2.032502365112305
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,32767,2.0302303314208983
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,32767,2.0342784881591798
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,32767,2.0371551513671875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,32767,2.03643684387207
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,1,65535,0.5775008201599121
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,2,65535,0.5634304046630859
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,4,65535,0.5581952095031738
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,8,65535,0.5552320003509521
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,16,65535,0.5614416122436523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,32,65535,0.5620431900024414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,32,1,64,65535,0.5612624168395997
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,1,65535,4.059743881225586
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,2,65535,4.046342468261718
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,4,65535,4.039523315429688
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,8,65535,4.026691055297851
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,16,65535,4.032819366455078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,32,65535,4.035391998291016
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,32,1,64,65535,4.035268783569336
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,1,0.014696000516414643
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,1,0.013676799833774567
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,1,0.013670399785041809
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,1,0.01324480026960373
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,1,0.013247999548912048
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,1,0.013876800239086152
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,1,0.013247999548912048
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,1,0.020897600054740905
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,1,0.01947840005159378
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,1,0.019326399266719817
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,1,0.019075199961662292
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,1,0.018966400623321535
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,1,0.018972800672054292
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,1,0.018961599469184874
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,3,0.014827199280261993
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,3,0.013780799508094788
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,3,0.013767999410629273
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,3,0.013279999792575835
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,3,0.013363200426101684
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,3,0.01313759982585907
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,3,0.013220800459384919
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,3,0.020425599813461304
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,3,0.019606399536132812
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,3,0.01958560049533844
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,3,0.019275200366973878
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,3,0.01912800073623657
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,3,0.01913599967956543
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,3,0.019262400269508363
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,7,0.014444799721240997
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,7,0.013704000413417817
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,7,0.013460800051689148
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,7,0.013463999330997466
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,7,0.013246400654315949
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,7,0.013556799292564392
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,7,0.013331200182437896
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,7,0.0206496000289917
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,7,0.019633600115776063
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,7,0.01934719979763031
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,7,0.019252799451351166
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,7,0.019489599764347075
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,7,0.019326399266719817
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,7,0.019067199528217317
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,15,0.014532800018787383
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,15,0.013827200233936309
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,15,0.013915200531482697
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,15,0.013497599959373474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,15,0.013527999818325042
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,15,0.013707199692726135
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,15,0.01345600038766861
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,15,0.02080480009317398
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,15,0.020131200551986694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,15,0.019732800126075745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,15,0.019971199333667755
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,15,0.019806399941444397
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,15,0.01966879963874817
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,15,0.019595199823379518
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,31,0.014550399780273438
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,31,0.013678400218486786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,31,0.013894400000572205
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,31,0.01340479999780655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,31,0.013521599769592284
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,31,0.013953599333763122
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,31,0.013655999302864074
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,31,0.023000000417232512
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,31,0.02218720018863678
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,31,0.021899199485778807
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,31,0.02173279970884323
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,31,0.021558399498462676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,31,0.021715199947357176
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,31,0.021671999990940095
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,63,0.01512800008058548
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,63,0.014672000706195832
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,63,0.01390720009803772
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,63,0.014192000031471252
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,63,0.013766400516033173
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,63,0.014281600713729858
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,63,0.013875199854373932
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,63,0.02614560127258301
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,63,0.025033599138259886
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,63,0.024873599410057068
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,63,0.024902400374412537
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,63,0.02458080053329468
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,63,0.024659200012683867
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,63,0.024553599953651428
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,127,0.017070400714874267
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,127,0.016225600242614747
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,127,0.01565439999103546
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,127,0.015919999778270723
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,127,0.01542080044746399
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,127,0.01595360040664673
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,127,0.015782399475574492
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,127,0.03361760079860687
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,127,0.03263840079307556
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,127,0.03213919997215271
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,127,0.032225599884986876
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,127,0.032216000556945804
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,127,0.03224000036716461
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,127,0.03205280005931854
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,255,0.02075359970331192
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,255,0.019494399428367615
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,255,0.0192671999335289
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,255,0.019062399864196777
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,255,0.018515199422836304
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,255,0.01870400011539459
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,255,0.019014400243759156
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,255,0.05045120120048523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,255,0.04842880070209503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,255,0.04793919920921326
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,255,0.047332799434661864
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,255,0.046849599480628966
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,255,0.04745280146598816
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,255,0.047060799598693845
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,511,0.0381056010723114
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,511,0.029139199852943422
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,511,0.02475520074367523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,511,0.023647999763488768
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,511,0.02643040120601654
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,511,0.027995198965072632
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,511,0.02826400101184845
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,511,0.0974128007888794
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,511,0.09029920101165771
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,511,0.08710399866104127
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,511,0.0864512026309967
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,511,0.08899040222167968
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,511,0.09086239933967591
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,511,0.09144480228424072
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,1023,0.04910080134868622
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,1023,0.040622401237487796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,1023,0.03708159923553467
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,1023,0.035545599460601804
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,1023,0.03795520067214966
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,1023,0.03882560133934021
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,1023,0.039547199010849
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,1023,0.16311520338058472
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,1023,0.15591520071029663
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,1023,0.15291520357131957
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,1023,0.15169600248336793
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,1023,0.15393439531326295
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,1023,0.15521119832992553
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,1023,0.15525120496749878
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,2047,0.06775680184364319
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,2047,0.05899680256843567
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,2047,0.055264002084732054
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,2047,0.05328959822654724
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,2047,0.055004799365997316
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,2047,0.055585598945617674
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,2047,0.05562880039215088
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,2047,0.29111518859863283
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,2047,0.2842992067337036
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,2047,0.28104000091552733
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,2047,0.2799983978271484
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,2047,0.2816704034805298
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,2047,0.28280479907989503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,2047,0.2828255891799927
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,4095,0.1041152000427246
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,4095,0.09483039975166321
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,4095,0.09027519822120667
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,4095,0.08802559971809387
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,4095,0.09244319796562195
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,4095,0.09444479942321778
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,4095,0.09550399780273437
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,4095,0.5445551872253418
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,4095,0.5371935844421387
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,4095,0.5340496063232422
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,4095,0.5324255943298339
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,4095,0.5358287811279296
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,4095,0.5381807804107666
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,4095,0.5388031959533691
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,8191,0.1738271951675415
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,8191,0.16383999586105347
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,8191,0.15935519933700562
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,8191,0.15716960430145263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,8191,0.16305919885635375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,8191,0.16911360025405883
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,8191,0.16886240243911743
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,8191,1.0444543838500977
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,8191,1.038111972808838
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,8191,1.033676815032959
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,8191,1.0316960334777832
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,8191,1.0393888473510742
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,8191,1.0431407928466796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,8191,1.0436896324157714
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,16383,0.30944321155548093
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,16383,0.29805281162261965
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,16383,0.2923151969909668
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,16383,0.28987200260162355
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,16383,0.29617760181427
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,16383,0.30053279399871824
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,16383,0.30123679637908934
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,16383,2.043412780761719
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,16383,2.034596824645996
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,16383,2.031662368774414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,16383,2.028691291809082
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,16383,2.03799991607666
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,16383,2.0425039291381837
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,16383,2.043276786804199
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,1,32767,0.5835968017578125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,2,32767,0.5676608085632324
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,4,32767,0.5625455856323243
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,8,32767,0.5558720111846924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,16,32767,0.5627151966094971
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,32,32767,0.5672207832336426
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,64,1,64,32767,0.5690432071685791
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,1,32767,4.0593711853027346
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,2,32767,4.048876953125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,4,32767,4.029916763305664
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,8,32767,4.029235076904297
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,16,32767,4.034758377075195
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,32,32767,4.038065719604492
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,64,1,64,32767,4.038348770141601
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,1,0.016420799493789672
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,1,0.015545600652694702
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,1,0.014788800477981567
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,1,0.015028800070285796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,1,0.015094399452209473
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,1,0.01465280055999756
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,1,0.01515360027551651
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,1,0.022465600073337554
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,1,0.02096959948539734
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,1,0.021080000698566435
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,1,0.021236799657344818
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,1,0.02083359956741333
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,1,0.02091040015220642
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,1,0.020934399962425233
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,3,0.016680000722408293
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,3,0.01533920019865036
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,3,0.015390400588512421
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,3,0.015057599544525147
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,3,0.015132799744606018
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,3,0.014612799882888794
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,3,0.014724799990653991
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,3,0.022294400632381438
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,3,0.021436800062656403
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,3,0.020732800662517547
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,3,0.020742399990558623
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,3,0.020891200006008147
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,3,0.02112800031900406
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,3,0.020657600462436677
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,7,0.016612799465656282
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,7,0.01510400027036667
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,7,0.01530880033969879
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,7,0.015240000188350677
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,7,0.014934399724006652
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,7,0.01485760062932968
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,7,0.014900800585746766
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,7,0.022907200455665588
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,7,0.021422399580478667
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,7,0.021556800603866576
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,7,0.021401600539684297
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,7,0.021495999395847322
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,7,0.021195200085639954
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,7,0.021003200113773345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,15,0.016628800332546233
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,15,0.01518079936504364
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,15,0.015355199575424194
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,15,0.015212799608707427
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,15,0.01533920019865036
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,15,0.01496960073709488
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,15,0.015334400534629821
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,15,0.025203201174736022
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,15,0.02351360023021698
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,15,0.023632000386714935
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,15,0.023497599363327026
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,15,0.023476800322532652
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,15,0.023182399570941925
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,15,0.023585599660873414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,31,0.017030400037765504
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,31,0.015542399883270264
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,31,0.015583999454975128
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,31,0.015345600247383118
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,31,0.015539200603961944
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,31,0.014980800449848175
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,31,0.015345600247383118
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,31,0.027881601452827455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,31,0.026025599241256712
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,31,0.026199999451637267
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,31,0.026206400990486146
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,31,0.026211199164390565
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,31,0.025843200087547303
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,31,0.025947201251983642
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,63,0.017659200727939604
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,63,0.01589920073747635
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,63,0.016036799550056456
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,63,0.015512000024318694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,63,0.015990400314331056
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,63,0.015651200711727143
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,63,0.015992000699043274
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,63,0.034729599952697754
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,63,0.03246400058269501
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,63,0.03234399855136871
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,63,0.03225759863853454
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,63,0.032267200946807864
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,63,0.03214240074157715
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,63,0.03234399855136871
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,127,0.019860799610614776
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,127,0.017875200510025023
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,127,0.017764799296855927
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,127,0.017958399653434754
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,127,0.018371200561523436
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,127,0.017433600127696992
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,127,0.017547200620174407
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,127,0.05305119752883911
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,127,0.048091199994087216
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,127,0.04723680019378662
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,127,0.04662240147590637
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,127,0.046449598670005796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,127,0.04594559967517853
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,127,0.046587198972702026
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,255,0.026315200328826904
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,255,0.02232320010662079
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,255,0.02190079987049103
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,255,0.02128800004720688
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,255,0.02162880003452301
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,255,0.020793600380420683
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,255,0.02162880003452301
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,255,0.08722400069236755
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,255,0.08469920158386231
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,255,0.08348479866981506
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,255,0.0828432023525238
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,255,0.08275039792060852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,255,0.08260800242424012
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,255,0.08253120183944702
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,511,0.041065600514411923
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,511,0.03699040114879608
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,511,0.03664799928665161
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,511,0.035657599568367004
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,511,0.03558239936828613
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,511,0.03482399880886078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,511,0.035220798850059507
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,511,0.1558127999305725
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,511,0.15293600559234619
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,511,0.1512671947479248
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,511,0.15053759813308715
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,511,0.15079519748687745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,511,0.1505280017852783
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,511,0.15070719718933107
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,1023,0.05890719890594483
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,1023,0.05500800013542175
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,1023,0.05360320210456848
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,1023,0.05287200212478638
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,1023,0.05313119888305664
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,1023,0.05248960256576538
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,1023,0.05296639800071716
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,1023,0.2844288110733032
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,1023,0.2806384086608887
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,1023,0.27961599826812744
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,1023,0.2797951936721802
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,1023,0.27948479652404784
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,1023,0.2790719985961914
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,1023,0.2796272039413452
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,2047,0.09443680047988892
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,2047,0.09024479985237122
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,2047,0.08867520093917847
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,2047,0.08812959790229798
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,2047,0.08790879845619201
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,2047,0.08691999912261963
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,2047,0.08732479810714722
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,2047,0.5367392063140869
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,2047,0.5333407878875732
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,2047,0.5315023899078369
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,2047,0.5321951866149902
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,2047,0.5318912029266357
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,2047,0.5320784091949463
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,2047,0.5315375804901123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,4095,0.16197760105133058
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,4095,0.15730079412460327
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,4095,0.15582239627838135
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,4095,0.1559631943702698
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,4095,0.156278395652771
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,4095,0.1543552041053772
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,4095,0.15400960445404052
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,4095,1.0361248016357423
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,4095,1.033073616027832
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,4095,1.0316783905029296
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,4095,1.0312848091125488
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,4095,1.0318896293640136
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,4095,1.0314528465270996
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,4095,1.0322431564331054
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,8191,0.3083584070205688
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,8191,0.2982336044311523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,8191,0.2895711898803711
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,8191,0.2881184101104736
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,8191,0.28903200626373293
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,8191,0.2876431941986084
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,8191,0.28945279121398926
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,8191,2.0342016220092773
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,8191,2.0297407150268554
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,8191,2.0282751083374024
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,8191,2.029220771789551
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,8191,2.0288976669311523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,8191,2.0300624847412108
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,8191,2.0283487319946287
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,1,16383,0.5850800037384033
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,2,16383,0.5635200023651123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,4,16383,0.5572144031524658
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,8,16383,0.5553152084350585
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,16,16383,0.556825590133667
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,32,16383,0.557478380203247
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,128,1,64,16383,0.5571216106414795
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,1,16383,4.053547286987305
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,2,16383,4.046780776977539
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,4,16383,4.026700973510742
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,8,16383,4.028652954101562
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,16,16383,4.0276447296142575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,32,16383,4.02728157043457
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,128,1,64,16383,4.027987289428711
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,1,0.02516320049762726
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,1,0.023369599878787995
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,1,0.022103999555110932
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,1,0.022171199321746826
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,1,0.02229280024766922
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,1,0.022100800275802614
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,1,0.021755200624465943
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,1,0.03219839930534363
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,1,0.02877120077610016
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,1,0.028652799129486085
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,1,0.02784320116043091
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,1,0.027628800272941588
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,1,0.027830401062965394
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,1,0.028064000606536865
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,3,0.0244719997048378
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,3,0.022995199263095855
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,3,0.021931199729442595
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,3,0.022124800086021423
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,3,0.022259199619293214
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,3,0.021956799924373625
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,3,0.021803200244903564
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,3,0.032094401121139524
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,3,0.029145601391792297
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,3,0.028222399950027465
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,3,0.028424000740051268
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,3,0.02853280007839203
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,3,0.028033599257469177
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,3,0.027822399139404298
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,7,0.025095999240875244
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,7,0.02326720058917999
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,7,0.022086399793624877
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,7,0.022047999501228332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,7,0.021859200298786165
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,7,0.02184160053730011
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,7,0.021718400716781616
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,7,0.03516319990158081
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,7,0.03154560029506683
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,7,0.030513599514961243
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,7,0.03046880066394806
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,7,0.03034079968929291
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,7,0.030167999863624572
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,7,0.02979680001735687
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,15,0.025041601061820982
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,15,0.023470400273799895
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,15,0.02255360037088394
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,15,0.022494399547576906
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,15,0.022515200078487396
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,15,0.022206400334835053
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,15,0.021916800737380983
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,15,0.03904959857463837
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,15,0.03444640040397644
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,15,0.033416000008583066
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,15,0.0331743985414505
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,15,0.03327359855175018
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,15,0.03309440016746521
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,15,0.03278079926967621
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,31,0.02592160105705261
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,31,0.023475199937820435
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,31,0.02222079932689667
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,31,0.022387200593948366
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,31,0.02234559953212738
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,31,0.02234880030155182
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,31,0.02211360037326813
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,31,0.047086399793624875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,31,0.04106720089912415
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,31,0.03920960128307342
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,31,0.039001598954200745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,31,0.0387584000825882
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,31,0.038657599687576295
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,31,0.03866719901561737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,63,0.03196159899234772
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,63,0.024449600279331206
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,63,0.023054400086402894
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,63,0.02316800057888031
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,63,0.022655999660491942
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,63,0.022920000553131103
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,63,0.022697600722312927
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,63,0.06422560214996338
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,63,0.057631999254226685
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,63,0.05421280264854431
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,63,0.053609597682952884
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,63,0.05247359871864319
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,63,0.05229120254516602
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,63,0.051729601621627805
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,127,0.037934398651123045
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,127,0.03350560069084167
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,127,0.02807359993457794
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,127,0.027169600129127502
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,127,0.02748799920082092
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,127,0.02712959945201874
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,127,0.026366400718688964
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,127,0.09784319996833801
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,127,0.09412639737129211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,127,0.09171680212020875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,127,0.08991360068321227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,127,0.08936319947242737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,127,0.08928160071372986
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,127,0.08911359906196595
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,255,0.04746240079402923
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,255,0.0430512011051178
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,255,0.04120480120182037
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,255,0.039854401350021364
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,255,0.039740800857543945
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,255,0.039084801077842714
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,255,0.03890239894390106
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,255,0.16244959831237793
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,255,0.15848480463027953
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,255,0.1571295976638794
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,255,0.15611519813537597
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,255,0.15581120252609254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,255,0.15508480072021485
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,255,0.15490399599075316
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,511,0.07247040271759034
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,511,0.06532800197601318
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,511,0.061515200138092044
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,511,0.060057598352432254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,511,0.05912320017814636
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,511,0.05899680256843567
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,511,0.059278398752212524
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,511,0.2990367889404297
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,511,0.29223198890686036
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,511,0.28912959098815916
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,511,0.287934398651123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,511,0.2868623971939087
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,511,0.2870192050933838
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,511,0.2869999885559082
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,1023,0.10772639513015747
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,1023,0.10022560358047486
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,1023,0.09588639736175537
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,1023,0.09416159987449646
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,1023,0.09359520077705383
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,1023,0.09329919815063477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,1023,0.09287199974060059
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,1023,0.5494815826416015
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,1023,0.5434832096099853
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,1023,0.5401919841766357
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,1023,0.537936019897461
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,1023,0.5375872135162354
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,1023,0.5373871803283692
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,1023,0.5374127864837647
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,2047,0.1754191994667053
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,2047,0.16700639724731445
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,2047,0.16276479959487916
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,2047,0.16056480407714843
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,2047,0.16028159856796265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,2047,0.15970560312271118
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,2047,0.15960639715194702
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,2047,1.0493280410766601
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,2047,1.0419631958007813
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,2047,1.0389504432678223
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,2047,1.0371552467346192
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,2047,1.0365632057189942
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,2047,1.036500835418701
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,2047,1.036409568786621
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,4095,0.31620159149169924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,4095,0.30133919715881347
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,4095,0.29699680805206297
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,4095,0.29484798908233645
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,4095,0.29443519115447997
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,4095,0.29443359375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,4095,0.29385440349578856
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,4095,2.047750473022461
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,4095,2.0414623260498046
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,4095,2.036793518066406
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,4095,2.035788726806641
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,4095,2.0353103637695313
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,4095,2.0362384796142576
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,4095,2.034943962097168
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,1,8191,0.6150320053100586
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,2,8191,0.6034031867980957
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,4,8191,0.5670544147491455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,8,8191,0.5640160083770752
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,16,8191,0.5621024131774902
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,32,8191,0.561624002456665
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,256,1,64,8191,0.5617760181427002
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,1,8191,4.06214714050293
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,2,8191,4.06384162902832
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,4,8191,4.036222457885742
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,8,8191,4.0299217224121096
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,16,8191,4.029734420776367
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,32,8191,4.029127883911133
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,256,1,64,8191,4.031684875488281
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,1,0.050380802154541014
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,1,0.03946560025215149
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,1,0.03746080100536346
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,1,0.036827200651168825
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,1,0.03672640025615692
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,1,0.0365119993686676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,1,0.03622399866580963
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,1,0.05760319828987122
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,1,0.046670401096343996
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,1,0.04386079907417297
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,1,0.04288159906864166
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,1,0.0427727997303009
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,1,0.04288800060749054
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,1,0.042654401063919066
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,3,0.04993599951267243
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,3,0.03904159963130951
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,3,0.03751200139522552
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,3,0.03668000102043152
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,3,0.03634720146656036
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,3,0.03639520108699799
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,3,0.03624480068683624
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,3,0.05974879860877991
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,3,0.049595201015472413
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,3,0.04623199999332428
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,3,0.045019200444221495
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,3,0.04453440010547638
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,3,0.04444159865379334
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,3,0.044363200664520264
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,7,0.04990400075912475
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,7,0.03930079936981201
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,7,0.03773280084133148
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,7,0.03676320016384125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,7,0.03645919859409332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,7,0.036380800604820254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,7,0.036268800497055054
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,7,0.06277120113372803
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,7,0.05299519896507263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,7,0.048161599040031436
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,7,0.04742560088634491
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,7,0.047547200322151185
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,7,0.04736959934234619
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,7,0.04708960056304932
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,15,0.05040320158004761
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,15,0.04000000059604645
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,15,0.03796960115432739
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,15,0.03707520067691803
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,15,0.036881598830223086
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,15,0.03679839968681335
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,15,0.03673279881477356
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,15,0.07048799991607665
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,15,0.06125440001487732
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,15,0.05567200183868408
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,15,0.05360320210456848
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,15,0.053513598442077634
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,15,0.05351999998092651
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,15,0.05322880148887634
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,31,0.051551997661590576
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,31,0.04177919924259186
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,31,0.03813759982585907
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,31,0.0372655987739563
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,31,0.0369951993227005
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,31,0.036748799681663516
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,31,0.03665600121021271
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,31,0.08481760025024414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,31,0.07582240104675293
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,31,0.07143359780311584
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,31,0.0678063988685608
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,31,0.06605920195579529
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,31,0.06588159799575806
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,31,0.06568959951400757
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,63,0.0563103973865509
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,63,0.04768959879875183
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,63,0.04316959977149963
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,63,0.040310400724411014
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,63,0.039001598954200745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,63,0.038278400897979736
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,63,0.038011199235916136
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,63,0.11766879558563233
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,63,0.10901119709014892
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,63,0.10535039901733398
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,63,0.10253440141677857
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,63,0.10063680410385131
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,63,0.10049599409103394
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,63,0.10034079551696777
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,127,0.06555039882659912
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,127,0.057580798864364624
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,127,0.053667199611663816
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,127,0.05201759934425354
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,127,0.050779199600219725
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,127,0.05054559707641602
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,127,0.04992960095405578
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,127,0.1816383957862854
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,127,0.1743664026260376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,127,0.17033920288085938
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,127,0.16795040369033815
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,127,0.16627199649810792
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,127,0.16584320068359376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,127,0.16605600118637084
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,255,0.08341280221939087
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,255,0.07506080269813538
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,255,0.0721072018146515
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,255,0.06976640224456787
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,255,0.06889119744300842
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,255,0.06758880019187927
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,255,0.06757439970970154
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,255,0.3072319984436035
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,255,0.29983360767364503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,255,0.2968991994857788
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,255,0.2948415994644165
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,255,0.2935296058654785
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,255,0.29293599128723147
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,255,0.2925087928771973
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,511,0.13551679849624634
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,511,0.12059520483016968
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,511,0.11285279989242554
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,511,0.10880800485610961
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,511,0.10767359733581543
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,511,0.10691519975662231
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,511,0.10678240060806274
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,511,0.5773168087005616
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,511,0.5633423805236817
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,511,0.5565375804901123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,511,0.5522719860076905
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,511,0.5516736030578613
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,511,0.5506912231445312
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,511,0.5500800132751464
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,1023,0.20374560356140137
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,1023,0.1881775975227356
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,1023,0.18027839660644532
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,1023,0.17594560384750366
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,1023,0.1746832013130188
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,1023,0.17398879528045655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,1023,0.17355200052261352
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,1023,1.0781999588012696
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,1023,1.0638064384460448
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,1023,1.056604766845703
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,1023,1.0521072387695312
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,1023,1.0514255523681642
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,1023,1.0500880241394044
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,1023,1.0497247695922851
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,2047,0.3431999921798706
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,2047,0.32208800315856934
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,2047,0.31382720470428466
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,2047,0.30937440395355226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,2047,0.30771679878234864
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,2047,0.3064591884613037
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,2047,0.30617120265960696
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,2047,2.074939155578613
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,2047,2.0621456146240233
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,2047,2.0545183181762696
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,2047,2.0491552352905273
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,2047,2.046886444091797
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,2047,2.0460351943969726
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,2047,2.0455583572387694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,1,4095,0.6390975952148438
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,2,4095,0.6142000198364258
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,4,4095,0.5837168216705322
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,8,4095,0.5765632152557373
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,16,4095,0.575438404083252
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,32,4095,0.5734960079193115
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,512,1,64,4095,0.5729904174804688
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,1,4095,4.072201538085937
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,2,4095,4.0648143768310545
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,4,4095,4.050774383544922
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,8,4095,4.0439807891845705
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,16,4095,4.04222412109375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,32,4095,4.043203353881836
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,512,1,64,4095,4.042828750610352
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,1,0.0873583972454071
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,1,0.07308800220489502
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,1,0.06460959911346435
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,1,0.061729598045349124
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,1,0.06119040250778198
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,1,0.060759997367858885
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,1,0.06104159951210022
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,1,0.09694399833679199
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,1,0.0826367974281311
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,1,0.07515040040016174
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,1,0.07071520090103149
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,1,0.06998720169067382
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,1,0.06944159865379333
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,1,0.06916319727897643
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,3,0.08730400204658509
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,3,0.07238240242004394
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,3,0.06461920142173767
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,3,0.06190239787101746
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,3,0.061345601081848146
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,3,0.06107680201530456
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,3,0.06091359853744507
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,3,0.10097279548645019
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,3,0.08625919818878174
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,3,0.0798255980014801
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,3,0.07334880232810974
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,3,0.07239199876785278
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,3,0.07251359820365906
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,3,0.07241759896278381
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,7,0.08865280151367187
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,7,0.07256799936294556
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,7,0.06419680118560792
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,7,0.061964797973632815
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,7,0.06120160222053528
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,7,0.06120799779891968
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,7,0.060811197757720946
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,7,0.10870399475097656
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,7,0.09269440174102783
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,7,0.08691520094871522
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,7,0.08038880228996277
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,7,0.07820960283279418
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,7,0.07845119833946228
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,7,0.07765439748764039
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,15,0.08925439715385437
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,15,0.07290080189704895
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,15,0.06601759791374207
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,15,0.061977601051330565
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,15,0.061768001317977904
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,15,0.06133279800415039
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,15,0.061273598670959474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,15,0.12367360591888428
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,15,0.10684000253677368
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,15,0.10156160593032837
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,15,0.09676160216331482
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,15,0.09372000098228454
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,15,0.09246399998664856
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,15,0.0907151997089386
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,31,0.09126240015029907
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,31,0.07459840178489685
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,31,0.06828960180282592
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,31,0.06378880143165588
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,31,0.0625872015953064
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,31,0.06138560175895691
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,31,0.06155679821968078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,31,0.15360959768295288
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,31,0.13798240423202515
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,31,0.13005440235137938
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,31,0.12613120079040527
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,31,0.1256384015083313
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,31,0.12481600046157837
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,31,0.12440160512924195
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,63,0.09879519939422607
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,63,0.08078399896621705
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,63,0.0737280011177063
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,63,0.07121440172195434
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,63,0.07025279998779296
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,63,0.06885759830474854
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,63,0.06862080097198486
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,63,0.21534080505371095
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,63,0.19817759990692138
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,63,0.190939199924469
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,63,0.18698240518569947
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,63,0.18602880239486694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,63,0.18544479608535766
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,63,0.18511840105056762
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,127,0.11669759750366211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,127,0.09859679937362671
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,127,0.0913856029510498
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,127,0.08733599781990051
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,127,0.08513280153274536
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,127,0.084443199634552
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,127,0.08412320017814637
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,127,0.3411072015762329
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,127,0.32436161041259765
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,127,0.31595358848571775
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,127,0.31133759021759033
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,127,0.30927839279174807
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,127,0.30847198963165284
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,127,0.3085792064666748
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,255,0.14818559885025023
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,255,0.13324960470199584
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,255,0.1260143995285034
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,255,0.12313280105590821
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,255,0.12064160108566284
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,255,0.11938879489898682
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,255,0.11844960451126099
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,255,0.5880288124084473
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,255,0.573748779296875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,255,0.5669888019561767
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,255,0.5636112213134765
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,255,0.5616352081298828
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,255,0.5610991954803467
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,255,0.5602416038513184
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,511,0.25086400508880613
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,511,0.22181758880615235
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,511,0.20681920051574706
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,511,0.20001120567321778
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,511,0.19664160013198853
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,511,0.19611519575119019
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,511,0.19480960369110106
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,511,1.1264800071716308
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,511,1.0954943656921388
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,511,1.0815919876098632
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,511,1.0738847732543946
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,511,1.0715087890625
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,511,1.0698960304260254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,511,1.0691984176635743
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,1023,0.3917520046234131
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,1023,0.3603008031845093
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,1023,0.3438976049423218
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,1023,0.33487839698791505
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,1023,0.332039999961853
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,1023,0.32988801002502444
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,1023,0.32809441089630126
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,1023,2.1253280639648438
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,1023,2.0980607986450197
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,1023,2.0817440032958983
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,1023,2.074679946899414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,1023,2.070919990539551
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,1023,2.068084716796875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,1023,2.0682704925537108
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,1,2047,0.6847216129302979
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,2,2047,0.6489232063293457
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,4,2047,0.6101984024047852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,8,2047,0.6005712032318116
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,16,2047,0.5966527938842774
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,32,2047,0.5947487831115723
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,1,1024,1,64,2047,0.5929183959960938
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,1,2047,4.1395313262939455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,2,2047,4.120537567138672
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,4,2047,4.0785888671875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,8,2047,4.069087982177734
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,16,2047,4.06628646850586
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,32,2047,4.063036727905273
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,1,1024,1,64,2047,4.0621185302734375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,1,0.012561599910259246
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,1,0.012169600278139115
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,1,0.011695999652147293
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,1,0.011531200259923935
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,1,0.011505600064992905
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,1,0.01154400035738945
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,1,0.011529599875211715
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,1,0.01834080070257187
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,1,0.01796319931745529
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,1,0.017601600289344786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,1,0.01743679940700531
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,1,0.017047999799251555
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,1,0.01717280000448227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,1,0.017260800302028655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,3,0.012833599746227265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,3,0.012296000123023986
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,3,0.011934400349855424
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,3,0.011524800211191177
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,3,0.011513599753379821
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,3,0.011494400352239609
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,3,0.01154400035738945
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,3,0.018367999792099
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,3,0.017851200699806214
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,3,0.017584000527858735
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,3,0.017510400712490083
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,3,0.01720000058412552
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,3,0.01713760048151016
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,3,0.017127999663352968
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,7,0.012740799784660339
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,7,0.01231520026922226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,7,0.011959999799728394
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,7,0.011587200313806533
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,7,0.011383999884128571
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,7,0.011559999734163284
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,7,0.011603199690580369
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,7,0.018492799997329713
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,7,0.017956799268722533
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,7,0.017419199645519256
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,7,0.017340800166130065
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,7,0.017236800491809846
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,7,0.0173007994890213
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,7,0.017334400117397307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,15,0.012715199589729309
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,15,0.012281599640846252
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,15,0.011876799911260606
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,15,0.011668799817562104
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,15,0.01165120005607605
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,15,0.011659199744462967
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,15,0.011646399646997452
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,15,0.018479999899864197
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,15,0.017927999794483184
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,15,0.017552000284194947
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,15,0.01743839979171753
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,15,0.017350399494171144
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,15,0.01735360026359558
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,15,0.017297600209712983
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,31,0.012779200077056884
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,31,0.012388800084590913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,31,0.011876799911260606
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,31,0.011659199744462967
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,31,0.011444800347089768
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,31,0.011631999909877778
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,31,0.011483199894428253
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,31,0.018408000469207764
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,31,0.018086400628089905
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,31,0.017643199861049653
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,31,0.017334400117397307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,31,0.017182399332523347
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,31,0.01738879978656769
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,31,0.017217600345611574
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,63,0.01279360055923462
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,63,0.012438400089740754
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,63,0.012052799761295318
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,63,0.011843200027942657
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,63,0.011646399646997452
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,63,0.011649599671363831
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,63,0.011593600362539291
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,63,0.018303999304771425
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,63,0.018187199532985688
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,63,0.01785600036382675
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,63,0.01765599995851517
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,63,0.01728159934282303
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,63,0.017569600045681
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,63,0.01744000017642975
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,127,0.01424960047006607
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,127,0.013971200585365296
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,127,0.013726399838924408
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,127,0.013516800105571746
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,127,0.013358399271965027
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,127,0.013473600149154663
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,127,0.01345440000295639
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,127,0.020136000216007234
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,127,0.01993280053138733
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,127,0.019492800533771514
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,127,0.019398400187492372
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,127,0.019174399971961974
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,127,0.0192208006978035
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,127,0.019236800074577332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,255,0.01749120056629181
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,255,0.01722240000963211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,255,0.016777600347995757
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,255,0.016527999937534333
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,255,0.01641920059919357
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,255,0.016505600512027742
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,255,0.01650079935789108
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,255,0.023263999819755556
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,255,0.023137600719928743
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,255,0.022543999552726745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,255,0.022383999824523926
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,255,0.022300800681114195
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,255,0.022176000475883483
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,255,0.022280000150203705
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,511,0.018350400030612946
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,511,0.018012799322605133
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,511,0.01684959977865219
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,511,0.016187199950218202
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,511,0.01568640023469925
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,511,0.01685280054807663
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,511,0.0170864000916481
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,511,0.02420639991760254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,511,0.0238319993019104
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,511,0.022699199616909027
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,511,0.022166399657726286
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,511,0.021768000721931458
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,511,0.02274879962205887
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,511,0.022912000119686127
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,1023,0.018982400000095368
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,1023,0.018236799538135527
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,1023,0.017081600427627564
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,1023,0.016392000019550323
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,1023,0.016196799278259278
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,1023,0.016918399930000307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,1023,0.01713760048151016
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,1023,0.02504960000514984
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,1023,0.02446399927139282
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,1023,0.02342880070209503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,1023,0.022884799540042876
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,1023,0.022475199401378633
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,1023,0.02333440035581589
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,1023,0.023580799996852874
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,2047,0.02072480022907257
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,2047,0.018904000520706177
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,2047,0.0177279993891716
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,2047,0.017110399901866913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,2047,0.016888000071048737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,2047,0.017283199727535246
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,2047,0.01733600050210953
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,2047,0.028479999303817748
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,2047,0.02757920026779175
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,2047,0.026096001267433167
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,2047,0.02550719976425171
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,2047,0.025187200307846068
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,2047,0.02547839879989624
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,2047,0.025684800744056702
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,4095,0.022070400416851044
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,4095,0.021348799765110015
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,4095,0.019539199769496918
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,4095,0.018676799535751343
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,4095,0.01865759938955307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,4095,0.019313600659370423
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,4095,0.019495999813079833
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,4095,0.032839998602867126
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,4095,0.03211359977722168
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,4095,0.031067198514938353
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,4095,0.029980799555778502
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,4095,0.02972959876060486
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,4095,0.03060159981250763
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,4095,0.030750399827957152
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,8191,0.02647840082645416
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,8191,0.02335200011730194
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,8191,0.021966400742530822
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,8191,0.021803200244903564
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,8191,0.02099519968032837
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,8191,0.022438399493694305
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,8191,0.022771200537681578
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,8191,0.043721601366996765
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,8191,0.04121119976043701
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,8191,0.0395823985338211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,8191,0.038441601395607
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,8191,0.03822399973869324
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,8191,0.039724799990653994
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,8191,0.0400191992521286
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,16383,0.0347216010093689
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,16383,0.027854400873184203
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,16383,0.026241600513458252
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,16383,0.024659200012683867
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,16383,0.024614399671554564
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,16383,0.024903999269008638
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,16383,0.024876800179481507
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,16383,0.06790239810943603
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,16383,0.06133120059967041
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,16383,0.05551360249519348
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,16383,0.05404959917068482
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,16383,0.053198397159576416
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,16383,0.05337439775466919
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,16383,0.053527998924255374
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,32767,0.04646719992160797
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,32767,0.0381632000207901
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,32767,0.031760001182556154
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,32767,0.02985759973526001
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,32767,0.029791998863220214
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,32767,0.02887519896030426
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,32767,0.028620800375938414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,32767,0.10690399408340454
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,32767,0.09998080134391785
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,32767,0.09440640211105347
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,32767,0.09318400025367737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,32767,0.09309759736061096
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,32767,0.09251840114593506
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,32767,0.09248639941215515
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,65535,0.061577600240707395
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,65535,0.048721599578857425
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,65535,0.043459200859069826
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,65535,0.04224480092525482
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,65535,0.04199999868869782
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,65535,0.04110080003738403
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,65535,0.03994719982147217
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,65535,0.17581599950790405
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,65535,0.16188000440597533
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,65535,0.16022239923477172
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,65535,0.15953600406646729
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,65535,0.15872160196304322
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,65535,0.15848000049591066
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,65535,0.15841280221939086
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1,1,1,131071,0.09196959733963013
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1,1,2,131071,0.06662560105323792
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1,1,4,131071,0.06018400192260742
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1,1,8,131071,0.058171200752258304
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1,1,16,131071,0.059617602825164796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1,1,32,131071,0.05691519975662231
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1,1,64,131071,0.05854719877243042
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1,1,1,131071,0.3124847888946533
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1,1,2,131071,0.28997600078582764
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1,1,4,131071,0.28825280666351316
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1,1,8,131071,0.286296010017395
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1,1,16,131071,0.28656320571899413
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1,1,32,131071,0.2856640100479126
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1,1,64,131071,0.2852832078933716
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,1,0.012694400548934937
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,1,0.01234399974346161
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,1,0.011843200027942657
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,1,0.011587200313806533
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,1,0.011377599835395814
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,1,0.011320000141859054
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,1,0.011444800347089768
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,1,0.018063999712467194
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,1,0.017800000309944154
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,1,0.0173007994890213
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,1,0.016967999935150146
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,1,0.017127999663352968
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,1,0.016966399550437928
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,1,0.016883200407028197
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,3,0.012505599856376648
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,3,0.012331199645996094
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,3,0.011800000071525573
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,3,0.011767999827861786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,3,0.011558400094509124
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,3,0.011572799831628799
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,3,0.011510399729013443
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,3,0.01791359931230545
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,3,0.017603200674057008
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,3,0.017369599640369417
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,3,0.01717440038919449
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,3,0.017049600183963776
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,3,0.017057600617408752
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,3,0.01703840047121048
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,7,0.012452799826860428
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,7,0.012195199728012085
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,7,0.011694400012493134
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,7,0.011580800265073776
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,7,0.01148959994316101
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,7,0.011540800333023071
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,7,0.011665599793195725
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,7,0.018137599527835845
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,7,0.01777759939432144
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,7,0.01736319959163666
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,7,0.017231999337673186
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,7,0.017067199945449828
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,7,0.01700640022754669
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,7,0.017215999960899352
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,15,0.01271200031042099
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,15,0.012571200728416443
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,15,0.011803200095891952
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,15,0.011790399998426437
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,15,0.011577600240707397
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,15,0.011577600240707397
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,15,0.011561600118875503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,15,0.018092800676822663
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,15,0.01785759925842285
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,15,0.017392000555992125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,15,0.01740639954805374
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,15,0.01730719953775406
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,15,0.016991999745368958
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,15,0.01706559956073761
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,31,0.012647999823093415
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,31,0.012457600235939026
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,31,0.011999999731779098
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,31,0.011772800236940384
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,31,0.011673600226640702
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,31,0.011617600172758102
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,31,0.011556799709796905
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,31,0.018147200345993042
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,31,0.01793439984321594
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,31,0.017479999363422392
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,31,0.017131200432777403
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,31,0.01725600063800812
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,31,0.017254400253295898
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,31,0.017166399955749513
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,63,0.01260959953069687
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,63,0.012415999919176102
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,63,0.01197120025753975
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,63,0.011748799681663513
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,63,0.011750400066375732
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,63,0.01173280030488968
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,63,0.011817599833011627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,63,0.017983999848365784
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,63,0.01817920058965683
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,63,0.017742399871349335
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,63,0.017473599314689635
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,63,0.01723040044307709
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,63,0.01727360039949417
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,63,0.017467199265956877
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,127,0.014315199851989747
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,127,0.014300799369812012
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,127,0.01369439959526062
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,127,0.013527999818325042
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,127,0.013382400572299957
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,127,0.013369600474834441
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,127,0.013376000523567199
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,127,0.019683200120925903
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,127,0.01980479955673218
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,127,0.019215999543666838
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,127,0.01897120028734207
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,127,0.01900160014629364
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,127,0.019068799912929535
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,127,0.0190080001950264
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,255,0.017262400686740877
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,255,0.017260800302028655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,255,0.01680160015821457
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,255,0.01679680049419403
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,255,0.016649599373340606
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,255,0.01652639955282211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,255,0.016513599455356597
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,255,0.022873599827289582
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,255,0.02304159998893738
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,255,0.022622400522232057
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,255,0.022312000393867493
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,255,0.022270399332046508
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,255,0.022099199891090392
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,255,0.0222448006272316
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,511,0.018726399540901183
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,511,0.017923200130462648
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,511,0.016761599481105803
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,511,0.016238400340080263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,511,0.015846399962902068
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,511,0.016873599588871004
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,511,0.017075200378894807
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,511,0.024952000379562377
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,511,0.02428479939699173
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,511,0.022833600640296936
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,511,0.02236959934234619
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,511,0.02192160040140152
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,511,0.02295680046081543
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,511,0.023131200671195985
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,1023,0.019415999948978423
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,1023,0.018644799292087556
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,1023,0.01725600063800812
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,1023,0.01676799952983856
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,1023,0.016286399960517884
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,1023,0.017166399955749513
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,1023,0.017310400307178498
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,1023,0.02770879864692688
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,1023,0.02688480019569397
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,1023,0.025430399179458617
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,1023,0.024694399535655977
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,1023,0.024102400243282317
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,1023,0.025060799717903138
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,1023,0.02542400062084198
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,2047,0.021158400177955627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,2047,0.01923999935388565
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,2047,0.01781439930200577
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,2047,0.01735839992761612
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,2047,0.017076799273490907
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,2047,0.017428800463676453
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,2047,0.01748799979686737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,2047,0.032529601454734804
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,2047,0.03054719865322113
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,2047,0.028984001278877257
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,2047,0.02802720069885254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,2047,0.02789439857006073
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,2047,0.028305599093437196
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,2047,0.028403198719024657
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,4095,0.02558239996433258
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,4095,0.021547199785709382
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,4095,0.01968639940023422
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,4095,0.01899999976158142
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,4095,0.01855040043592453
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,4095,0.019174399971961974
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,4095,0.019868800044059755
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,4095,0.04248160123825073
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,4095,0.03903360068798065
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,4095,0.036689600348472594
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,4095,0.03597759902477264
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,4095,0.035559999942779544
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,4095,0.036284801363945005
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,4095,0.03643999993801117
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,8191,0.03175680041313171
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,8191,0.026414400339126586
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,8191,0.022401599586009978
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,8191,0.02174399942159653
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,8191,0.021289600431919097
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,8191,0.023871999979019166
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,8191,0.024622400104999543
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,8191,0.0665120005607605
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,8191,0.059747201204299924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,8191,0.052104002237319945
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,8191,0.05095999836921692
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,8191,0.05021759867668152
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,8191,0.05263360142707825
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,8191,0.053046399354934694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,16383,0.042798399925231934
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,16383,0.03619680106639862
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,16383,0.02908959984779358
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,16383,0.026321598887443544
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,16383,0.025750398635864258
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,16383,0.027750399708747864
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,16383,0.028043198585510253
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,16383,0.10326240062713624
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,16383,0.09698879718780518
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,16383,0.09104160070419312
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,16383,0.08931360244750977
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,16383,0.08938239812850952
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,16383,0.09098560214042664
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,16383,0.09170560240745544
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,32767,0.05785120129585266
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,32767,0.05014079809188843
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,32767,0.040057599544525146
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,32767,0.03749440014362335
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,32767,0.03676480054855347
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,32767,0.03849599957466125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,32767,0.03887679874897003
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,32767,0.17374240159988402
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,32767,0.1634335994720459
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,32767,0.15541759729385377
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,32767,0.1550927996635437
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,32767,0.1538480043411255
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,32767,0.15601439476013185
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,32767,0.156496000289917
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,65535,0.08679680228233337
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,65535,0.06469759941101075
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,65535,0.0561519980430603
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,65535,0.054604798555374146
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,65535,0.05353760123252869
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,65535,0.054996800422668454
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,65535,0.05624799728393555
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,65535,0.30999040603637695
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,65535,0.2876287937164307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,65535,0.28454880714416503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,65535,0.28382720947265627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,65535,0.28317599296569823
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,65535,0.28516159057617185
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,65535,0.2853872060775757
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,2,1,1,131071,0.14552160501480102
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,2,1,2,131071,0.09799360036849976
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,2,1,4,131071,0.0925328016281128
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,2,1,8,131071,0.09085919857025146
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,2,1,16,131071,0.0894544005393982
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,2,1,32,131071,0.09129760265350342
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,2,1,64,131071,0.0910256028175354
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,2,1,1,131071,0.5836991786956787
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,2,1,2,131071,0.53995680809021
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,2,1,4,131071,0.5369503974914551
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,2,1,8,131071,0.5356815814971924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,2,1,16,131071,0.5348368167877198
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,2,1,32,131071,0.5362048149108887
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,2,1,64,131071,0.5361408233642578
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,1,0.012747199833393097
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,1,0.012249600142240524
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,1,0.01196800023317337
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,1,0.01175680011510849
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,1,0.011617600172758102
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,1,0.011460799723863602
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,1,0.011590400338172912
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,1,0.018268799781799315
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,1,0.017972800135612487
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,1,0.0174127995967865
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,1,0.01722559928894043
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,1,0.017081600427627564
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,1,0.017238399386405943
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,1,0.017235200107097625
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,3,0.01265919953584671
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,3,0.012375999987125397
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,3,0.011912000179290772
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,3,0.0118367999792099
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,3,0.011660800129175187
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,3,0.011611200124025344
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,3,0.011640000343322753
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,3,0.018163199722766876
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,3,0.017931200563907623
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,3,0.01743839979171753
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,3,0.017339199781417847
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,3,0.016990399360656737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,3,0.017118400335311888
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,3,0.01714719980955124
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,7,0.012780800461769104
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,7,0.012464000284671784
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,7,0.01199679970741272
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,7,0.011796800047159195
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,7,0.011572799831628799
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,7,0.011432000249624253
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,7,0.011611200124025344
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,7,0.018217599391937254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,7,0.018041600286960603
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,7,0.017478400468826295
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,7,0.01723040044307709
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,7,0.01704320013523102
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,7,0.017156800627708434
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,7,0.01717280000448227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,15,0.012694400548934937
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,15,0.012479999661445617
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,15,0.012043199688196182
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,15,0.01189119964838028
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,15,0.011724799871444702
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,15,0.01168000027537346
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,15,0.011687999963760376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,15,0.018169599771499633
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,15,0.017824000120162962
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,15,0.017472000420093538
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,15,0.01740960031747818
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,15,0.017243200540542604
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,15,0.017235200107097625
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,15,0.017385600507259368
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,31,0.012726399302482604
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,31,0.012491200119256973
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,31,0.011902400106191636
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,31,0.011691199988126755
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,31,0.011745599657297134
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,31,0.011687999963760376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,31,0.011857599765062333
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,31,0.01839520037174225
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,31,0.018078400194644927
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,31,0.017577600479125977
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,31,0.017416000366210938
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,31,0.01727360039949417
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,31,0.017312000691890716
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,31,0.017396800220012665
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,63,0.012788799405097962
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,63,0.012625600397586822
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,63,0.012091200053691863
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,63,0.011849600076675414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,63,0.011675199866294861
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,63,0.011627200245857238
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,63,0.011745599657297134
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,63,0.018361599743366243
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,63,0.018254399299621582
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,63,0.017819200456142426
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,63,0.017425599694252013
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,63,0.017340800166130065
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,63,0.01751199960708618
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,63,0.017398400604724883
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,127,0.01435839980840683
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,127,0.014201599359512328
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,127,0.013716800510883332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,127,0.013583999872207642
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,127,0.01343040019273758
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,127,0.013353599607944489
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,127,0.01361600011587143
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,127,0.020160000026226043
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,127,0.01998720020055771
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,127,0.019366399943828584
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,127,0.01950719952583313
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,127,0.019079999625682832
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,127,0.01908479928970337
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,127,0.019180800020694732
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,255,0.01767359972000122
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,255,0.017449599504470826
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,255,0.017049600183963776
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,255,0.016758400201797485
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,255,0.016596800088882445
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,255,0.01656000018119812
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,255,0.016457599401474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,255,0.02341119945049286
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,255,0.023499199748039247
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,255,0.022937600314617158
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,255,0.022705599665641785
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,255,0.0227743998169899
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,255,0.022657600045204163
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,255,0.02244960069656372
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,511,0.019659200310707094
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,511,0.018615999817848207
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,511,0.017220799624919892
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,511,0.016470399498939515
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,511,0.016233600676059723
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,511,0.017228800058364867
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,511,0.01754560023546219
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,511,0.027849599719047546
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,511,0.02698720097541809
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,511,0.025521600246429445
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,511,0.024582399427890776
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,511,0.024348799884319306
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,511,0.025297600030899047
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,511,0.02542079985141754
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,1023,0.021982400119304656
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,1023,0.019283199310302736
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,1023,0.017684799432754517
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,1023,0.016924799978733064
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,1023,0.016655999422073364
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,1023,0.017372800409793852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,1023,0.017403200268745422
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,1023,0.03260320127010345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,1023,0.030158400535583496
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,1023,0.02879199981689453
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,1023,0.027684798836708067
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,1023,0.027531200647354127
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,1023,0.028262400627136232
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,1023,0.02842719852924347
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,2047,0.02563839852809906
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,2047,0.021425600349903106
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,2047,0.019395199418067933
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,2047,0.01785600036382675
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,2047,0.017822399735450745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,2047,0.017931200563907623
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,2047,0.01812160015106201
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,2047,0.042217600345611575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,2047,0.03850240111351013
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,2047,0.03580160140991211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,2047,0.03491039872169495
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,2047,0.03454560041427612
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,2047,0.03484640121459961
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,2047,0.034974399209022525
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,4095,0.030462399125099182
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,4095,0.025911998748779298
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,4095,0.021772800385951994
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,4095,0.020294399559497835
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,4095,0.020049600303173064
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,4095,0.021342399716377258
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,4095,0.02144960016012192
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,4095,0.06404320001602173
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,4095,0.05939840078353882
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,4095,0.05209760069847107
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,4095,0.04960800111293793
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,4095,0.04875679910182953
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,4095,0.04972479939460754
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,4095,0.05035200119018555
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,8191,0.03980000019073486
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,8191,0.034353598952293396
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,8191,0.027039998769760133
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,8191,0.024937599897384644
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,8191,0.023948800563812257
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,8191,0.02736319899559021
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,8191,0.028336000442504884
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,8191,0.10032479763031006
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,8191,0.09572799801826477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,8191,0.09000639915466309
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,8191,0.08791040182113648
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,8191,0.08758720159530639
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,8191,0.0917136013507843
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,8191,0.09216960072517395
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,16383,0.05475999712944031
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,16383,0.04684320092201233
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,16383,0.04008319973945618
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,16383,0.03575200140476227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,16383,0.034536001086235044
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,16383,0.037975999712944034
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,16383,0.03888800144195557
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,16383,0.17023999691009523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,16383,0.16135200262069702
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,16383,0.15387840270996095
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,16383,0.15315840244293213
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,16383,0.15244319438934326
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,16383,0.15569599866867065
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,16383,0.1568671941757202
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,32767,0.08324959874153137
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,32767,0.06305119991302491
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,32767,0.054895997047424316
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,32767,0.0531216025352478
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,32767,0.05403839945793152
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,32767,0.05688639879226685
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,32767,0.05777599811553955
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,32767,0.3071808099746704
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,32767,0.2872159957885742
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,32767,0.283351993560791
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,32767,0.28227360248565675
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,32767,0.2810368061065674
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,32767,0.2842063903808594
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,32767,0.28516640663146975
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,65535,0.14088640213012696
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,65535,0.0967631995677948
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,65535,0.09025920033454896
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,65535,0.0877568006515503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,65535,0.08735039830207825
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,65535,0.09022560119628906
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,65535,0.09104160070419312
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,65535,0.5794335842132569
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,65535,0.5374767780303955
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,65535,0.5333712100982666
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,65535,0.5329311847686767
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,65535,0.5318304061889648
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,65535,0.5354368209838867
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,65535,0.5357808113098145
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,4,1,1,131071,0.25438239574432375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,4,1,2,131071,0.1652351975440979
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,4,1,4,131071,0.16025439500808716
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,4,1,8,131071,0.1580288052558899
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,4,1,16,131071,0.15702079534530639
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,4,1,32,131071,0.16022399663925171
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,4,1,64,131071,0.1602959990501404
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,4,1,1,131071,1.12117919921875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,4,1,2,131071,1.0389039993286133
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,4,1,4,131071,1.0358160018920899
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,4,1,8,131071,1.0335472106933594
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,4,1,16,131071,1.0332063674926757
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,4,1,32,131071,1.035865592956543
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,4,1,64,131071,1.037168025970459
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,1,0.012932799756526947
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,1,0.01252799928188324
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,1,0.011934400349855424
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,1,0.011878400295972823
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,1,0.011750400066375732
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,1,0.011903999745845795
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,1,0.011736000329256058
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,1,0.018760000169277192
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,1,0.018038399517536163
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,1,0.01796800047159195
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,1,0.017504000663757326
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,1,0.017529599368572235
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,1,0.017416000366210938
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,1,0.017556799948215483
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,3,0.012883199751377106
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,3,0.012559999525547028
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,3,0.011872000247240066
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,3,0.011710400134325028
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,3,0.011558400094509124
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,3,0.011572799831628799
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,3,0.01154559999704361
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,3,0.018639999628067016
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,3,0.01833280026912689
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,3,0.017825600504875184
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,3,0.0173567995429039
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,3,0.017496000230312347
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,3,0.017560000717639922
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,3,0.017529599368572235
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,7,0.013363200426101684
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,7,0.012513600289821625
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,7,0.012041600048542022
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,7,0.011868800222873687
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,7,0.01165279969573021
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,7,0.01175519973039627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,7,0.011700800061225891
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,7,0.018699200451374055
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,7,0.018225599825382233
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,7,0.018027199804782866
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,7,0.017745600640773775
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,7,0.017636799812316896
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,7,0.017451199889183044
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,7,0.017591999471187593
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,15,0.012859199941158295
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,15,0.012827199697494508
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,15,0.012177599966526032
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,15,0.012089599668979645
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,15,0.011819200217723846
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,15,0.011921600252389909
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,15,0.011843200027942657
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,15,0.019444799423217772
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,15,0.018268799781799315
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,15,0.0176816001534462
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,15,0.017871999740600587
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,15,0.017894400656223296
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,15,0.017761600017547608
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,15,0.017548799514770508
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,31,0.013451200723648072
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,31,0.012825599312782288
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,31,0.01223680004477501
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,31,0.012176000326871873
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,31,0.011711999773979187
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,31,0.011934400349855424
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,31,0.011838400363922119
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,31,0.019009600579738616
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,31,0.01857600063085556
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,31,0.017817600071430205
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,31,0.017712000012397765
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,31,0.017564800381660462
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,31,0.017688000202178956
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,31,0.017793600261211396
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,63,0.013049599528312684
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,63,0.012667199969291687
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,63,0.01210239976644516
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,63,0.012124799937009812
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,63,0.011854399740695954
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,63,0.011883199959993363
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,63,0.011899200081825257
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,63,0.01887200027704239
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,63,0.01868479996919632
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,63,0.018219199776649476
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,63,0.017897599935531618
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,63,0.018203200399875642
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,63,0.01788319945335388
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,63,0.01798879951238632
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,127,0.014640000462532044
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,127,0.0146479994058609
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,127,0.013942399621009826
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,127,0.013798399269580841
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,127,0.0135903999209404
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,127,0.013596799969673157
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,127,0.013569599390029908
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,127,0.02072799950838089
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,127,0.02078399956226349
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,127,0.020203199982643128
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,127,0.019870400428771973
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,127,0.01979680061340332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,127,0.020059199631214143
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,127,0.01977279931306839
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,255,0.01780160069465637
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,255,0.017579199373722078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,255,0.01709440052509308
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,255,0.01687840074300766
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,255,0.016763199865818024
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,255,0.016991999745368958
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,255,0.016700799763202667
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,255,0.025775998830795288
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,255,0.025655999779701233
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,255,0.02520799934864044
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,255,0.025158399343490602
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,255,0.024723200500011443
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,255,0.024849599599838255
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,255,0.024716800451278685
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,511,0.021433599293231964
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,511,0.019377599656581878
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,511,0.017550399899482726
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,511,0.016974399983882903
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,511,0.016457599401474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,511,0.017560000717639922
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,511,0.017641599476337432
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,511,0.03226400017738342
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,511,0.030351999402046203
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,511,0.028892800211906433
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,511,0.027911999821662904
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,511,0.02760159969329834
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,511,0.02875039875507355
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,511,0.028732800483703615
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,1023,0.024476799368858337
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,1023,0.021817600727081297
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,1023,0.018512000143527985
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,1023,0.017627200484275816
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,1023,0.017340800166130065
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,1023,0.01820639967918396
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,1023,0.018078400194644927
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,1023,0.0417136013507843
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,1023,0.038878399133682254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,1023,0.035550400614738464
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,1023,0.03467200100421906
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,1023,0.034302398562431335
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,1023,0.035099199414253233
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,1023,0.03518239855766296
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,2047,0.02895359992980957
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,2047,0.02563199996948242
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,2047,0.02197439968585968
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,2047,0.020057600736618043
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,2047,0.018904000520706177
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,2047,0.019867199659347533
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,2047,0.020121599733829498
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,2047,0.06338080167770385
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,2047,0.059115201234817505
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,2047,0.05170080065727234
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,2047,0.04939840137958527
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,2047,0.048511999845504764
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,2047,0.048583999276161194
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,2047,0.0488319993019104
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,4095,0.03717280030250549
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,4095,0.03257760107517242
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,4095,0.027718400955200194
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,4095,0.02456959933042526
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,4095,0.023337599635124207
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,4095,0.02526240050792694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,4095,0.025364801287651062
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,4095,0.09782879948616027
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,4095,0.09121440052986145
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,4095,0.08727679848670959
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,4095,0.08507999777793884
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,4095,0.08460320234298706
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,4095,0.08594239950180053
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,4095,0.08623200058937072
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,8191,0.05106239914894104
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,8191,0.04347679913043976
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,8191,0.039078399538993835
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,8191,0.034939199686050415
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,8191,0.033302399516105655
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,8191,0.03775680065155029
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,8191,0.03826079964637756
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,8191,0.16698559522628784
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,8191,0.158297598361969
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,8191,0.15445120334625245
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,8191,0.15266239643096924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,8191,0.15179840326309205
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,8191,0.15606080293655394
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,8191,0.15712640285491944
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,16383,0.07821599841117859
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,16383,0.061540800333023074
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,16383,0.05464800000190735
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,16383,0.05218719840049744
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,16383,0.05119360089302063
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,16383,0.05538880228996277
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,16383,0.05627520084381103
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,16383,0.3012592077255249
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,16383,0.2866528034210205
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,16383,0.2825216054916382
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,16383,0.2809839963912964
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,16383,0.28006079196929934
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,16383,0.2846463918685913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,16383,0.28533120155334474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,32767,0.13332959413528442
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,32767,0.09757279753684997
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,32767,0.09089919924736023
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,32767,0.08804320096969605
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,32767,0.08652960062026978
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,32767,0.09114400148391724
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,32767,0.09170879721641541
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,32767,0.5723487854003906
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,32767,0.5396175861358643
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,32767,0.5348800182342529
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,32767,0.5325520038604736
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,32767,0.5320256233215332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,32767,0.5361839771270752
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,32767,0.5369919776916504
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,65535,0.24412961006164552
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,65535,0.1658031940460205
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,65535,0.16023199558258056
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,65535,0.15764319896697998
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,65535,0.1560639977455139
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,65535,0.15947200059890748
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,65535,0.16106879711151123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,65535,1.1108736038208007
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,65535,1.0386159896850586
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,65535,1.0351455688476563
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,65535,1.0329423904418946
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,65535,1.0322015762329102
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,65535,1.0363360404968263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,65535,1.0374048233032227
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,8,1,1,131071,0.4711599826812744
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,8,1,2,131071,0.30438079833984377
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,8,1,4,131071,0.29555039405822753
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,8,1,8,131071,0.291974401473999
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,8,1,16,131071,0.2907151937484741
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,8,1,32,131071,0.2955296039581299
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,8,1,64,131071,0.29639039039611814
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,8,1,1,131071,2.187099266052246
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,8,1,2,131071,2.0383663177490234
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,8,1,4,131071,2.0352336883544924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,8,1,8,131071,2.0303071975708007
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,8,1,16,131071,2.0323535919189455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,8,1,32,131071,2.0357088088989257
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,8,1,64,131071,2.0370319366455076
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,1,0.01393280029296875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,1,0.013048000633716583
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,1,0.01231520026922226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,1,0.011767999827861786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,1,0.01189119964838028
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,1,0.01165120005607605
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,1,0.011843200027942657
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,1,0.01924159973859787
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,1,0.018211199343204497
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,1,0.017843200266361235
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,1,0.017560000717639922
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,1,0.017257599532604216
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,1,0.017691199481487275
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,1,0.017558400332927705
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,3,0.013716800510883332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,3,0.012918399274349212
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,3,0.012228800356388092
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,3,0.01178399994969368
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,3,0.011681599915027619
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,3,0.011627200245857238
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,3,0.01178399994969368
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,3,0.019332799315452575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,3,0.01815200001001358
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,3,0.017664000391960144
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,3,0.017534400522708892
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,3,0.0174127995967865
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,3,0.01746399998664856
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,3,0.017395199835300447
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,7,0.013803200423717498
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,7,0.012919999659061432
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,7,0.012377600371837615
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,7,0.011955200135707856
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,7,0.011913599818944931
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,7,0.011648000031709672
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,7,0.011926399916410447
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,7,0.019108800590038298
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,7,0.018479999899864197
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,7,0.017771199345588684
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,7,0.017481599748134614
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,7,0.01737920045852661
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,7,0.017423999309539796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,7,0.01722240000963211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,15,0.01377439945936203
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,15,0.01276479959487915
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,15,0.012425599992275238
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,15,0.012006399780511856
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,15,0.012148799747228623
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,15,0.011907199770212174
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,15,0.012147200107574464
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,15,0.019412800669670105
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,15,0.018555200099945067
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,15,0.018199999630451203
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,15,0.017505599558353423
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,15,0.01751199960708618
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,15,0.017539200186729432
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,15,0.01754560023546219
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,31,0.013785600662231445
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,31,0.013177600502967835
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,31,0.012403199821710587
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,31,0.012121599912643433
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,31,0.011846400052309036
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,31,0.01188960000872612
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,31,0.012268800288438797
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,31,0.019652800261974336
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,31,0.018433600664138794
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,31,0.01799200028181076
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,31,0.018067200481891633
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,31,0.017667199671268462
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,31,0.017875200510025023
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,31,0.017764799296855927
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,63,0.014124800264835358
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,63,0.013097600638866424
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,63,0.01241919994354248
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,63,0.012027200311422348
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,63,0.01244639977812767
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,63,0.012435200065374375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,63,0.012212800234556198
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,63,0.02019200026988983
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,63,0.01974879950284958
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,63,0.018555200099945067
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,63,0.018334400653839112
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,63,0.01839520037174225
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,63,0.018326400220394133
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,63,0.018244799971580506
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,127,0.015703999996185304
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,127,0.014683200418949128
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,127,0.014395199716091156
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,127,0.01393119990825653
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,127,0.013913600146770478
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,127,0.014188799262046813
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,127,0.01395840048789978
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,127,0.023416000604629516
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,127,0.022648000717163087
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,127,0.022096000611782074
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,127,0.021657599508762358
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,127,0.021782399713993074
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,127,0.021755200624465943
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,127,0.02170239984989166
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,255,0.018739199638366698
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,255,0.01775359958410263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,255,0.01706079989671707
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,255,0.01733759939670563
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,255,0.017387199401855468
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,255,0.01685439944267273
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,255,0.017073599994182585
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,255,0.02951520085334778
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,255,0.02861280143260956
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,255,0.02815999984741211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,255,0.02778240144252777
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,255,0.027540799975395203
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,255,0.027614399790763855
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,255,0.02778559923171997
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,511,0.02582719922065735
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,511,0.021515199542045595
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,511,0.018972800672054292
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,511,0.017900800704956053
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,511,0.017166399955749513
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,511,0.018006399273872375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,511,0.018438400328159334
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,511,0.04332799911499023
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,511,0.03809120059013367
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,511,0.03598879873752594
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,511,0.03452959954738617
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,511,0.034031999111175534
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,511,0.03487519919872284
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,511,0.03504000008106232
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,1023,0.030561599135398864
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,1023,0.024721600115299225
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,1023,0.02154559940099716
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,1023,0.019952000677585603
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,1023,0.019358399510383605
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,1023,0.020529599487781526
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,1023,0.021427200734615327
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,1023,0.06458560228347779
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,1023,0.05814560055732727
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,1023,0.051907199621200564
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,1023,0.049511998891830444
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,1023,0.04818080067634582
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,1023,0.04892959892749786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,1023,0.04984480142593384
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,2047,0.03838239908218384
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,2047,0.030990400910377504
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,2047,0.027059200406074523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,2047,0.02476319968700409
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,2047,0.023257599771022798
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,2047,0.024187199771404266
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,2047,0.02481119930744171
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,2047,0.09893599748611451
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,2047,0.09212639927864075
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,2047,0.08968480229377747
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,2047,0.0872816026210785
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,2047,0.08634880185127258
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,2047,0.08706560134887695
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,2047,0.08734080195426941
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,4095,0.05204960107803345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,4095,0.042070400714874265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,4095,0.037555199861526486
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,4095,0.03550719916820526
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,4095,0.033287999033927915
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,4095,0.035343998670578004
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,4095,0.03675679862499237
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,4095,0.1676095962524414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,4095,0.15580960512161254
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,4095,0.15364160537719726
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,4095,0.1514672040939331
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,4095,0.1505936026573181
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,4095,0.1524415969848633
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,4095,0.15324640274047852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,8191,0.07895519733428955
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,8191,0.06118879914283752
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,8191,0.055851197242736815
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,8191,0.053127998113632204
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,8191,0.0514240026473999
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,8191,0.05611199736595154
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,8191,0.05655360221862793
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,8191,0.30347840785980223
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,8191,0.2850800037384033
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,8191,0.2821343898773193
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,8191,0.27994399070739745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,8191,0.2789344072341919
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,8191,0.28299360275268554
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,8191,0.28438560962677
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,16383,0.13483359813690185
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,16383,0.09668480157852173
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,16383,0.09080320000648498
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,16383,0.08842880129814149
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,16383,0.0862496018409729
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,16383,0.09178079962730408
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,16383,0.09259520173072815
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,16383,0.5743328094482422
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,16383,0.5384975910186768
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,16383,0.5334879875183105
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,16383,0.5326767921447754
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,16383,0.5316927909851075
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,16383,0.5362143993377686
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,16383,0.5363359928131104
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,32767,0.24482240676879882
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,32767,0.1657039999961853
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,32767,0.1596351981163025
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,32767,0.1568735957145691
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,32767,0.15473599433898927
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,32767,0.16000640392303467
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,32767,0.1602720022201538
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,32767,1.1120464324951171
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,32767,1.037217617034912
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,32767,1.0335552215576171
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,32767,1.0316368103027345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,32767,1.0315088272094726
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,32767,1.035483169555664
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,32767,1.0363696098327637
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,65535,0.472544002532959
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,65535,0.30178720951080323
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,65535,0.2934704065322876
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,65535,0.29142398834228517
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,65535,0.2878592014312744
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,65535,0.2918240070343018
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,65535,0.29341280460357666
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,65535,2.1901391983032226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,65535,2.036310386657715
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,65535,2.0308528900146485
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,65535,2.0278175354003904
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,65535,2.029083251953125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,65535,2.032246398925781
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,65535,2.033430480957031
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,16,1,1,131071,0.9271535873413086
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,16,1,2,131071,0.5756752014160156
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,16,1,4,131071,0.5705135822296142
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,16,1,8,131071,0.5577248096466064
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,16,1,16,131071,0.5573056221008301
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,16,1,32,131071,0.5619952201843261
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,16,1,64,131071,0.5613264083862305
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,16,1,1,131071,4.368708801269531
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,16,1,2,131071,4.050468826293946
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,16,1,4,131071,4.0290672302246096
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,16,1,8,131071,4.027062225341797
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,16,1,16,131071,4.02549934387207
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,16,1,32,131071,4.032368087768555
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,16,1,64,131071,4.032486343383789
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,1,0.014211200177669525
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,1,0.013499200344085693
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,1,0.012804800271987915
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,1,0.012715199589729309
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,1,0.012600000202655792
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,1,0.012724800407886505
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,1,0.012643200159072877
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,1,0.019937600195407867
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,1,0.019273599982261656
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,1,0.018571199476718904
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,1,0.018486399948596955
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,1,0.018220800161361694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,1,0.018216000497341157
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,1,0.01847359985113144
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,3,0.014268800616264343
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,3,0.013910399377346038
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,3,0.012862400710582733
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,3,0.012567999958992004
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,3,0.012468799948692322
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,3,0.012566399574279786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,3,0.012651200592517852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,3,0.020499199628829956
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,3,0.019280000030994414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,3,0.018726399540901183
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,3,0.018561600148677825
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,3,0.018278400599956512
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,3,0.018270400166511536
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,3,0.018302400410175324
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,7,0.014313599467277527
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,7,0.013950400054454803
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,7,0.013128000497817992
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,7,0.013016000390052795
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,7,0.012668800354003907
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,7,0.012863999605178833
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,7,0.012747199833393097
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,7,0.020457600057125092
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,7,0.019526399672031403
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,7,0.01876640021800995
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,7,0.018428799510002137
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,7,0.018454399704933167
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,7,0.018432000279426576
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,7,0.01844639927148819
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,15,0.014528000354766845
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,15,0.014105600118637086
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,15,0.013033600151538849
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,15,0.01282079964876175
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,15,0.012681600451469422
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,15,0.012777599692344665
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,15,0.012583999335765839
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,15,0.020102399587631225
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,15,0.019651199877262115
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,15,0.018915200233459474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,15,0.018580800294876097
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,15,0.018760000169277192
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,15,0.01854880005121231
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,15,0.018753600120544434
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,31,0.01441120058298111
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,31,0.01414719969034195
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,31,0.013606399297714233
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,31,0.012972800433635712
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,31,0.012734399735927581
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,31,0.012857599556446076
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,31,0.012748800218105316
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,31,0.020720000565052032
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,31,0.020417599380016326
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,31,0.019065600633621217
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,31,0.019011199474334717
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,31,0.0191648006439209
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,31,0.018987199664115904
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,31,0.01880960017442703
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,63,0.014846399426460266
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,63,0.014569599926471711
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,63,0.013604800403118133
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,63,0.013198399543762207
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,63,0.013391999900341034
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,63,0.013169600069522858
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,63,0.012960000336170197
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,63,0.022654399275779724
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,63,0.02255840003490448
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,63,0.021507200598716737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,63,0.02119999974966049
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,63,0.02106720060110092
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,63,0.021171200275421142
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,63,0.02096959948539734
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,127,0.01660960018634796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,127,0.015999999642372132
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,127,0.015107199549674988
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,127,0.015089599788188935
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,127,0.014828799664974213
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,127,0.015201599895954132
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,127,0.01478399932384491
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,127,0.02719680070877075
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,127,0.026919999718666078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,127,0.025884801149368288
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,127,0.025798401236534117
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,127,0.02542079985141754
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,127,0.02566240131855011
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,127,0.025577598810195924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,255,0.019283199310302736
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,255,0.019201600551605226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,255,0.018294399976730345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,255,0.01815840005874634
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,255,0.018001599609851836
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,255,0.018292799592018127
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,255,0.018169599771499633
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,255,0.036103999614715575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,255,0.03596319854259491
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,255,0.03463680148124695
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,255,0.03440319895744324
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,255,0.03434720039367676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,255,0.03456319868564606
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,255,0.034483200311660765
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,511,0.034248000383377074
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,511,0.026601600646972656
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,511,0.021723200380802155
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,511,0.020764799416065217
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,511,0.019460800290107726
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,511,0.021065600216388702
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,511,0.021991999447345735
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,511,0.06813759803771972
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,511,0.06059839725494385
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,511,0.05308640003204346
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,511,0.049979200959205626
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,511,0.04841119945049286
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,511,0.049619200825691226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,511,0.05044800043106079
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,1023,0.04193600118160248
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,1023,0.033046400547027587
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,1023,0.026447999477386474
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,1023,0.024454399943351746
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,1023,0.023496000468730925
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,1023,0.026235198974609374
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,1023,0.027689599990844728
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,1023,0.10285600423812866
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,1023,0.093367999792099
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,1023,0.08763999938964843
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,1023,0.08642719984054566
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,1023,0.08569440245628357
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,1023,0.08738240003585815
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,1023,0.08831359744071961
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,2047,0.05574880242347717
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,2047,0.04373759925365448
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,2047,0.036795198917388916
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,2047,0.03532640039920807
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,2047,0.03336319923400879
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,2047,0.03494080007076263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,2047,0.03574239909648895
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,2047,0.17120959758758544
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,2047,0.15770720243453978
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,2047,0.1533903956413269
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,2047,0.15201120376586913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,2047,0.1505743980407715
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,2047,0.15154240131378174
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,2047,0.15242880582809448
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,4095,0.08443359732627868
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,4095,0.0631376028060913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,4095,0.056724798679351804
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,4095,0.05326240062713623
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,4095,0.051832002401351926
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,4095,0.054711997509002686
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,4095,0.055985599756240845
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,4095,0.30785279273986815
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,4095,0.28662400245666503
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,4095,0.2823823928833008
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,4095,0.2805327892303467
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,4095,0.2793936014175415
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,4095,0.2819375991821289
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,4095,0.28344318866729734
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,8191,0.1401568055152893
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,8191,0.09919040203094483
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,8191,0.09234399795532226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,8191,0.08870239853858948
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,8191,0.08718400001525879
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,8191,0.09274079799652099
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,8191,0.09630399942398071
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,8191,0.5786464214324951
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,8191,0.5391392230987548
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,8191,0.5350592136383057
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,8191,0.5325791835784912
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,8191,0.5313151836395263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,8191,0.5374368190765381
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,8191,0.5410160064697266
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,16383,0.2499295949935913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,16383,0.16776640415191652
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,16383,0.1617408037185669
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,16383,0.1575999975204468
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,16383,0.15628960132598876
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,16383,0.16073600053787232
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,16383,0.16415679454803467
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,16383,1.1147007942199707
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,16383,1.0381584167480469
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,16383,1.0352767944335937
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,16383,1.032639980316162
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,16383,1.0316864013671876
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,16383,1.03744478225708
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,16383,1.0379839897155763
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,32767,0.4761392116546631
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,32767,0.3041424036026001
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,32767,0.29506878852844237
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,32767,0.29224319458007814
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,32767,0.29027519226074217
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,32767,0.29620959758758547
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,32767,0.2952672004699707
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,32767,2.1920400619506837
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,32767,2.03787841796875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,32767,2.0329504013061523
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,32767,2.02917594909668
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,32767,2.0296703338623048
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,32767,2.0353023529052736
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,32767,2.03460807800293
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,32,1,1,65535,0.9332863807678222
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,32,1,2,65535,0.579036808013916
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,32,1,4,65535,0.572926378250122
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,32,1,8,65535,0.5578176021575928
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,32,1,16,65535,0.5543424129486084
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,32,1,32,65535,0.5636000156402587
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,32,1,64,65535,0.5599679946899414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,32,1,1,65535,4.376939010620117
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,32,1,2,65535,4.055203247070312
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,32,1,4,65535,4.042168045043946
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,32,1,8,65535,4.027515029907226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,32,1,16,65535,4.025449752807617
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,32,1,32,65535,4.031849670410156
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,32,1,64,65535,4.033115386962891
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,1,0.01576640009880066
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,1,0.014788800477981567
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,1,0.013649600744247436
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,1,0.013708800077438354
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,1,0.01356479972600937
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,1,0.013204799592494964
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,1,0.013596799969673157
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,1,0.021422399580478667
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,1,0.02027360051870346
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,1,0.019313600659370423
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,1,0.019312000274658202
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,1,0.019023999571800232
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,1,0.01878879964351654
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,1,0.018774400651454925
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,3,0.01578879952430725
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,3,0.01472959965467453
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,3,0.013911999762058258
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,3,0.013527999818325042
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,3,0.01342879980802536
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,3,0.013201600313186646
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,3,0.013211199641227722
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,3,0.02157440036535263
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,3,0.02003840059041977
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,3,0.0193231999874115
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,3,0.01919520050287247
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,3,0.018772800266742707
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,3,0.01907680034637451
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,3,0.0188511997461319
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,7,0.015600000321865082
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,7,0.014679999649524688
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,7,0.013734400272369385
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,7,0.013344000279903411
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,7,0.013663999736309052
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,7,0.013356800377368926
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,7,0.013607999682426453
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,7,0.021556800603866576
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,7,0.02003519982099533
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,7,0.019340799748897554
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,7,0.019355200231075287
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,7,0.01918880045413971
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,7,0.01937279999256134
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,7,0.019230400025844575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,15,0.0160303995013237
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,15,0.014396800100803376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,15,0.013865600526332855
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,15,0.013687999546527862
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,15,0.013329599797725678
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,15,0.013302400708198547
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,15,0.013651199638843536
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,15,0.02245440036058426
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,15,0.02115360051393509
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,15,0.019966399669647215
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,15,0.019891199469566346
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,15,0.01976960003376007
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,15,0.019431999325752257
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,15,0.019713599979877473
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,31,0.015876799821853638
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,31,0.015025599300861359
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,31,0.013784000277519226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,31,0.01348160058259964
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,31,0.013358399271965027
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,31,0.013475200533866883
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,31,0.013334399461746216
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,31,0.02425280064344406
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,31,0.023043200373649597
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,31,0.02211039960384369
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,31,0.021865600347518922
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,31,0.021639999747276307
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,31,0.021593600511550903
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,31,0.021755200624465943
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,63,0.01592639982700348
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,63,0.0154448002576828
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,63,0.01409280002117157
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,63,0.013814400136470794
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,63,0.014097599685192109
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,63,0.013672000169754029
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,63,0.013687999546527862
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,63,0.027055999636650084
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,63,0.026387199759483337
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,63,0.025121599435806274
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,63,0.025099200010299683
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,63,0.024907200038433074
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,63,0.024711999297142028
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,63,0.02476480007171631
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,127,0.017972800135612487
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,127,0.01684959977865219
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,127,0.016363200545310975
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,127,0.01611679941415787
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,127,0.01568319946527481
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,127,0.01584800034761429
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,127,0.015510399639606477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,127,0.03531680107116699
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,127,0.03365280032157898
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,127,0.0325872004032135
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,127,0.03230080008506775
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,127,0.03227199912071228
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,127,0.032201600074768064
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,127,0.03222880065441132
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,255,0.02110240012407303
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,255,0.020657600462436677
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,255,0.01950560063123703
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,255,0.019208000600337984
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,255,0.019211199879646302
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,255,0.018904000520706177
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,255,0.019131200015544893
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,255,0.055088001489639285
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,255,0.05026559829711914
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,255,0.04848639965057373
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,255,0.04782240092754364
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,255,0.04731360077857971
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,255,0.047091200947761536
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,255,0.04749439954757691
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,511,0.033976000547409055
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,511,0.03684160113334656
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,511,0.028615999221801757
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,511,0.02473919987678528
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,511,0.02354400008916855
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,511,0.026604801416397095
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,511,0.028310400247573853
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,511,0.09534239768981934
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,511,0.09716160297393799
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,511,0.0900272011756897
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,511,0.0870464026927948
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,511,0.08652319908142089
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,511,0.08905919790267944
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,511,0.09069439768791199
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,1023,0.047367998957633974
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,1023,0.04758560061454773
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,1023,0.039284801483154295
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,1023,0.03531039953231811
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,1023,0.03438720107078552
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,1023,0.03684960007667541
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,1023,0.038335999846458434
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,1023,0.16364480257034303
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,1023,0.16139999628067017
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,1023,0.1557263970375061
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,1023,0.15288959741592406
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,1023,0.15172319412231444
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,1023,0.15394560098648072
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,1023,0.15531680583953858
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,2047,0.07394559979438782
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,2047,0.06662560105323792
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,2047,0.058075201511383054
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,2047,0.05474879741668701
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,2047,0.05236319899559021
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,2047,0.05394880175590515
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,2047,0.05568320155143738
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,2047,0.2986639976501465
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,2047,0.2898303985595703
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,2047,0.2842047929763794
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,2047,0.2810960054397583
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,2047,0.27983999252319336
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,2047,0.2817039966583252
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,2047,0.2827552080154419
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,4095,0.13026560544967652
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,4095,0.10309920310974122
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,4095,0.09379839897155762
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,4095,0.0897328019142151
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,4095,0.0871504008769989
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,4095,0.09124159812927246
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,4095,0.09404640197753907
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,4095,0.5704800128936768
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,4095,0.5428800106048584
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,4095,0.5363999843597412
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,4095,0.5333759784698486
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,4095,0.5321199893951416
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,4095,0.5354496002197265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,4095,0.5377039909362793
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,8191,0.2411344051361084
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,8191,0.17428799867630004
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,8191,0.16437599658966065
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,8191,0.15919840335845947
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,8191,0.15611040592193604
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,8191,0.16302239894866943
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,8191,0.16867680549621583
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,8191,1.1088751792907714
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,8191,1.0452095985412597
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,8191,1.038148784637451
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,8191,1.0346159934997559
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,8191,1.033505630493164
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,8191,1.0396559715270997
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,8191,1.0439760208129882
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,16383,0.46871199607849123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,16383,0.3092447996139526
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,16383,0.2977247953414917
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,16383,0.2923968076705933
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,16383,0.2916399955749512
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,16383,0.29730560779571535
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,16383,0.3009632110595703
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,16383,2.185083198547363
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,16383,2.0436752319335936
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,16383,2.035152053833008
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,16383,2.0333440780639647
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,16383,2.030868721008301
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,16383,2.0365264892578123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,16383,2.0416208267211915
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,64,1,1,32767,0.9262080192565918
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,64,1,2,32767,0.5847072124481201
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,64,1,4,32767,0.5671040058135987
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,64,1,8,32767,0.5595583915710449
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,64,1,16,32767,0.5587056159973145
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,64,1,32,32767,0.5624383926391602
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,64,1,64,32767,0.5705808162689209
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,64,1,1,32767,4.366574478149414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,64,1,2,32767,4.058451080322266
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,64,1,4,32767,4.040932846069336
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,64,1,8,32767,4.029147338867188
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,64,1,16,32767,4.027598571777344
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,64,1,32,32767,4.033886337280274
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,64,1,64,32767,4.038710403442383
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,1,0.025216001272201537
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,1,0.016758400201797485
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,1,0.015252800285816192
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,1,0.015281599760055543
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,1,0.015193599462509155
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,1,0.015436799824237823
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,1,0.015006400644779205
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,1,0.030129599571228027
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,1,0.02223999947309494
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,1,0.021091200411319733
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,1,0.021247999370098115
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,1,0.021116800606250763
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,1,0.021003200113773345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,1,0.021065600216388702
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,3,0.02431679964065552
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,3,0.01650879979133606
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,3,0.015083199739456177
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,3,0.015169599652290344
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,3,0.015171200037002563
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,3,0.015460799634456634
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,3,0.014748799800872802
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,3,0.030929601192474364
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,3,0.022361600399017335
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,3,0.021353599429130555
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,3,0.0208064004778862
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,3,0.020776000618934632
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,3,0.020694400370121
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,3,0.021172800660133363
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,7,0.02499839961528778
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,7,0.016710400581359863
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,7,0.015195199847221374
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,7,0.015307199954986573
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,7,0.01517920047044754
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,7,0.014961600303649902
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,7,0.014856000244617463
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,7,0.032092800736427306
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,7,0.023027199506759643
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,7,0.02130880057811737
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,7,0.02136480063199997
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,7,0.02144159972667694
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,7,0.021428799629211424
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,7,0.02117920070886612
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,15,0.024758400022983552
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,15,0.016606399416923524
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,15,0.015084800124168397
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,15,0.015240000188350677
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,15,0.015169599652290344
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,15,0.015291200578212738
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,15,0.014936000108718872
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,15,0.03542239964008331
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,15,0.02507199943065643
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,15,0.023481599986553192
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,15,0.02373439967632294
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,15,0.02340639978647232
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,15,0.023366400599479677
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,15,0.023207999765872955
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,31,0.02515200078487396
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,31,0.01698399931192398
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,31,0.015388800203800202
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,31,0.01536639928817749
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,31,0.015289600193500518
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,31,0.015267199277877808
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,31,0.0147599995136261
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,31,0.03942880034446716
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,31,0.028139200806617738
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,31,0.02598879933357239
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,31,0.02620159983634949
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,31,0.02622399926185608
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,31,0.02603679895401001
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,31,0.025700798630714415
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,63,0.026732799410820008
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,63,0.01783040016889572
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,63,0.01587039977312088
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,63,0.016014400124549865
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,63,0.015824000537395477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,63,0.0160288006067276
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,63,0.01568160057067871
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,63,0.04660640060901642
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,63,0.0350735992193222
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,63,0.03253760039806366
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,63,0.0323168009519577
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,63,0.0322847992181778
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,63,0.032364800572395325
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,63,0.03197920024394989
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,127,0.034272000193595886
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,127,0.019577600061893463
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,127,0.017584000527858735
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,127,0.017696000635623932
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,127,0.017555199563503265
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,127,0.018193599581718446
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,127,0.01759680062532425
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,127,0.06614239811897278
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,127,0.05252640247344971
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,127,0.048286399245262145
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,127,0.047646400332450864
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,127,0.0477295994758606
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,127,0.04765760004520416
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,127,0.046697598695755
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,255,0.04135999977588654
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,255,0.02667199969291687
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,255,0.022059200704097746
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,255,0.022444799542427063
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,255,0.02184319943189621
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,255,0.021751999855041504
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,255,0.02048799991607666
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,255,0.10074399709701538
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,255,0.08734880089759826
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,255,0.08418400287628174
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,255,0.08329920172691345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,255,0.08266239762306213
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,255,0.08283519744873047
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,255,0.08249760270118714
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,511,0.06040639877319336
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,511,0.0397136002779007
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,511,0.03570559918880463
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,511,0.03430559933185577
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,511,0.03440800011157989
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,511,0.03375999927520752
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,511,0.033327999711036685
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,511,0.17738080024719238
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,511,0.15366079807281494
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,511,0.15264960527420043
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,511,0.1510815978050232
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,511,0.15086400508880615
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,511,0.15065280199050904
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,511,0.15079360008239745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,1023,0.08772320151329041
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,1023,0.057841598987579346
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,1023,0.05440800189971924
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,1023,0.052158397436141965
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,1023,0.05146880149841308
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,1023,0.05148640275001526
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,1023,0.05122399926185608
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,1023,0.31248960494995115
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,1023,0.2827807903289795
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,1023,0.2807984113693237
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,1023,0.27938079833984375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,1023,0.2791536092758179
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,1023,0.2788624048233032
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,1023,0.27900640964508056
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,2047,0.14198720455169678
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,2047,0.09327999949455261
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,2047,0.0889024019241333
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,2047,0.0874127984046936
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,2047,0.08639519810676574
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,2047,0.08629599809646607
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,2047,0.0862991988658905
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,2047,0.584003210067749
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,2047,0.5367504119873047
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,2047,0.5344143867492676
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,2047,0.5331759929656983
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,2047,0.5323200225830078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,2047,0.5309040069580078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,2047,0.5307055950164795
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,4095,0.25448639392852784
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,4095,0.17041440010070802
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,4095,0.15810240507125856
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,4095,0.1565359950065613
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,4095,0.15607520341873168
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,4095,0.15675840377807618
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,4095,0.15514559745788575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,4095,1.121395206451416
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,4095,1.0375311851501465
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,4095,1.0334303855895997
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,4095,1.0329919815063477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,4095,1.0316736221313476
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,4095,1.032204818725586
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,4095,1.0318960189819335
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,8191,0.481993579864502
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,8191,0.317248010635376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,8191,0.2997472047805786
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,8191,0.29095680713653566
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,8191,0.2892784118652344
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,8191,0.28845920562744143
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,8191,0.2904031991958618
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,8191,2.198369598388672
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,8191,2.039963150024414
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,8191,2.0324752807617186
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,8191,2.030971145629883
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,8191,2.0302431106567385
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,8191,2.0295759201049806
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,8191,2.0289119720458983
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,128,1,1,16383,0.9404656410217285
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,128,1,2,16383,0.589412784576416
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,128,1,4,16383,0.5649792194366455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,128,1,8,16383,0.5592160224914551
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,128,1,16,16383,0.5587327957153321
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,128,1,32,16383,0.5555344104766846
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,128,1,64,16383,0.5554080009460449
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,128,1,1,16383,4.386505508422852
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,128,1,2,16383,4.053305435180664
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,128,1,4,16383,4.035358428955078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,128,1,8,16383,4.025543975830078
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,128,1,16,16383,4.026204681396484
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,128,1,32,16383,4.026692962646484
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,128,1,64,16383,4.025067138671875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,1,0.048372799158096315
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,1,0.025051200389862062
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,1,0.023097600042819976
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,1,0.022259199619293214
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,1,0.02218559980392456
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,1,0.022155199944972993
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,1,0.022379200160503387
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,1,0.054872000217437746
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,1,0.03174079954624176
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,1,0.028836798667907716
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,1,0.028571200370788575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,1,0.02768000066280365
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,1,0.027739199995994567
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,1,0.027612799406051637
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,3,0.04784319996833801
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,3,0.024916799366474153
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,3,0.02339999973773956
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,3,0.022163200378417968
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,3,0.02213920056819916
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,3,0.02221439927816391
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,3,0.021849599480628968
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,3,0.05488799810409546
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,3,0.032148799300193785
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,3,0.02980000078678131
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,3,0.028435200452804565
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,3,0.028262400627136232
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,3,0.02850399911403656
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,3,0.0281360000371933
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,7,0.04750080108642578
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,7,0.0249551996588707
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,7,0.023231999576091768
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,7,0.022097599506378175
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,7,0.022059200704097746
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,7,0.02210240066051483
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,7,0.021857599914073943
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,7,0.05666080117225647
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,7,0.034867200255393985
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,7,0.03133600056171417
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,7,0.030396801233291627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,7,0.030251199007034303
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,7,0.030372801423072814
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,7,0.03006559908390045
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,15,0.04799199998378754
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,15,0.025360000133514405
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,15,0.023292799293994904
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,15,0.022519999742507936
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,15,0.022441600263118745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,15,0.022256000339984892
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,15,0.022033600509166716
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,15,0.05963199734687805
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,15,0.03866240084171295
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,15,0.034360000491142274
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,15,0.033318400382995605
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,15,0.033062401413917544
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,15,0.033055999875068666
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,15,0.03291040062904358
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,31,0.04806239902973175
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,31,0.02632000148296356
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,31,0.023078399896621703
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,31,0.022404800355434417
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,31,0.022379200160503387
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,31,0.022360000014305114
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,31,0.0221791997551918
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,31,0.06673280000686646
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,31,0.04684000015258789
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,31,0.040355199575424196
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,31,0.03893280029296875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,31,0.038812801241874695
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,31,0.038708800077438356
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,31,0.03872320055961609
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,63,0.0506496012210846
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,63,0.03164800107479095
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,63,0.0243136003613472
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,63,0.02284799963235855
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,63,0.022750400006771088
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,63,0.02260800004005432
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,63,0.022752000391483305
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,63,0.08390399813652039
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,63,0.06310719847679139
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,63,0.05778239965438843
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,63,0.0534991979598999
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,63,0.05244479775428772
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,63,0.05208799839019775
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,63,0.0518127977848053
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,127,0.05964319705963135
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,127,0.037567999958992
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,127,0.03204799890518188
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,127,0.02831200063228607
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,127,0.027214398980140685
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,127,0.027062401175498962
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,127,0.02683520019054413
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,127,0.11982719898223877
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,127,0.09617919921875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,127,0.09402239918708802
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,127,0.09168959856033325
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,127,0.08983839750289917
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,127,0.08928480148315429
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,127,0.08920320272445678
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,255,0.07203360199928284
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,255,0.04586080014705658
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,255,0.04187999963760376
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,255,0.03993600010871887
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,255,0.03866080045700073
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,255,0.03865599930286408
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,255,0.03823840022087097
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,255,0.18752800226211547
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,255,0.16246240139007567
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,255,0.15879839658737183
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,255,0.15718560218811034
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,255,0.15582879781723022
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,255,0.15580639839172364
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,255,0.15521279573440552
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,511,0.11212639808654785
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,511,0.07163519859313965
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,511,0.0640175998210907
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,511,0.06053439974784851
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,511,0.05897279977798462
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,511,0.057815998792648315
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,511,0.057652801275253296
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,511,0.33672959804534913
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,511,0.29795041084289553
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,511,0.29211199283599854
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,511,0.2890671968460083
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,511,0.2881824016571045
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,511,0.2868063926696777
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,511,0.28678240776062014
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,1023,0.1670912027359009
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,1023,0.1067247986793518
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,1023,0.09892640113830567
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,1023,0.09500799775123596
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,1023,0.09338560104370117
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,1023,0.09262880086898803
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,1023,0.0923807978630066
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,1023,0.6053631782531739
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,1023,0.5491519927978515
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,1023,0.5429791927337646
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,1023,0.5397535800933838
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,1023,0.5382095813751221
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,1023,0.5377535820007324
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,1023,0.5381296157836915
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,2047,0.27740960121154784
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,2047,0.17510559558868408
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,2047,0.1671471953392029
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,2047,0.16264159679412843
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,2047,0.16078399419784545
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,2047,0.1599392056465149
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,2047,0.16040639877319335
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,2047,1.1426783561706544
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,2047,1.0490079879760743
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,2047,1.0419440269470215
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,2047,1.0388671875
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,2047,1.0369296073913574
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,2047,1.0362303733825684
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,2047,1.036025619506836
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,4095,0.5099311828613281
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,4095,0.339352011680603
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,4095,0.30118560791015625
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,4095,0.29725279808044436
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,4095,0.29493279457092286
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,4095,0.2942608118057251
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,4095,0.29420158863067625
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,4095,2.222198486328125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,4095,2.0560735702514648
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,4095,2.040193557739258
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,4095,2.036564826965332
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,4095,2.03505916595459
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,4095,2.034846305847168
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,4095,2.0349376678466795
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,256,1,1,8191,0.970206356048584
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,256,1,2,8191,0.6323984146118165
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,256,1,4,8191,0.6028223991394043
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,256,1,8,8191,0.5665328025817871
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,256,1,16,8191,0.5621647834777832
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,256,1,32,8191,0.56080641746521
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,256,1,64,8191,0.5609039783477783
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,256,1,1,8191,4.406190490722656
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,256,1,2,8191,4.060945510864258
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,256,1,4,8191,4.046500778198242
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,256,1,8,8191,4.0329742431640625
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,256,1,16,8191,4.0303089141845705
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,256,1,32,8191,4.028849411010742
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,256,1,64,8191,4.028268814086914
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,1,0.0871936023235321
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,1,0.050040000677108766
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,1,0.03918879926204681
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,1,0.03735679984092712
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,1,0.03656800091266632
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,1,0.03653280138969421
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,1,0.03636159896850586
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,1,0.09392639994621277
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,1,0.057467198371887206
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,1,0.046619200706481935
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,1,0.043705600500106814
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,1,0.043089601397514346
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,1,0.042716801166534424
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,1,0.04264479875564575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,3,0.08568639755249023
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,3,0.04969440102577209
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,3,0.0392767995595932
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,3,0.037652799487113954
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,3,0.0367792010307312
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,3,0.03662559986114502
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,3,0.036422398686408994
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,3,0.0955295979976654
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,3,0.05958080291748047
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,3,0.04986560046672821
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,3,0.04591520130634308
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,3,0.04477120041847229
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,3,0.04490559995174408
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,3,0.04472000002861023
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,7,0.08630719780921936
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,7,0.049451199173927304
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,7,0.03917759954929352
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,7,0.03761920034885406
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,7,0.03687039911746979
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,7,0.03665600121021271
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,7,0.036668801307678224
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,7,0.09881439805030823
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,7,0.06302080154418946
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,7,0.05311520099639892
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,7,0.04856640100479126
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,7,0.04790880084037781
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,7,0.047547200322151185
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,7,0.0472463995218277
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,15,0.08674880266189575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,15,0.0495824009180069
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,15,0.039955198764801025
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,15,0.03792479932308197
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,15,0.03700000047683716
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,15,0.036959999799728395
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,15,0.036817601323127745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,15,0.10686880350112915
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,15,0.07000319957733155
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,15,0.061433601379394534
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,15,0.055446398258209226
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,15,0.053748798370361325
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,15,0.053527998924255374
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,15,0.0534496009349823
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,31,0.0897487998008728
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,31,0.05089600086212158
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,31,0.04175199866294861
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,31,0.037920001149177554
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,31,0.03707360029220581
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,31,0.03694080114364624
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,31,0.036883199214935304
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,31,0.12290400266647339
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,31,0.08313440084457398
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,31,0.07602400183677674
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,31,0.07160639762878418
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,31,0.06766719818115234
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,31,0.06632959842681885
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,31,0.0659600019454956
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,63,0.09172639846801758
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,63,0.055851197242736815
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,63,0.047111999988555905
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,63,0.04259839951992035
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,63,0.04020000100135803
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,63,0.03865599930286408
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,63,0.03816159963607788
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,63,0.15305919647216798
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,63,0.11537920236587525
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,63,0.1092960000038147
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,63,0.10515680313110351
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,63,0.10236959457397461
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,63,0.10119359493255616
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,63,0.10058079957962036
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,127,0.10492000579833985
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,127,0.06433280110359192
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,127,0.05658559799194336
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,127,0.05317760109901428
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,127,0.051123201847076416
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,127,0.05024319887161255
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,127,0.05034080147743225
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,127,0.21988160610198976
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,127,0.1791200041770935
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,127,0.1739776015281677
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,127,0.17017760276794433
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,127,0.16815199851989746
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,127,0.1664191961288452
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,127,0.16581439971923828
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,255,0.1309856057167053
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,255,0.08294720053672791
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,255,0.07470719814300537
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,255,0.07100160121917724
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,255,0.06899679899215698
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,255,0.06795359849929809
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,255,0.06704480051994324
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,255,0.3545664072036743
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,255,0.30671679973602295
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,255,0.299561595916748
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,255,0.2968496084213257
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,255,0.29531519412994384
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,255,0.29407200813293455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,255,0.2925312042236328
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,511,0.2136768102645874
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,511,0.13432320356369018
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,511,0.11938079595565795
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,511,0.11232320070266724
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,511,0.10748000144958496
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,511,0.10657279491424561
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,511,0.10572799444198608
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,511,0.6525167942047119
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,511,0.5773712158203125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,511,0.5630688190460205
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,511,0.5570144176483154
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,511,0.5524720191955567
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,511,0.5510608196258545
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,511,0.5507056236267089
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,1023,0.3232320070266724
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,1023,0.20675199031829833
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,1023,0.1883072018623352
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,1023,0.1809424042701721
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,1023,0.17630560398101808
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,1023,0.17495360374450683
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,1023,0.1738576054573059
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,1023,1.188758373260498
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,1023,1.0790800094604491
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,1023,1.0630784034729004
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,1023,1.0573871612548829
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,1023,1.0521151542663574
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,1023,1.051523208618164
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,1023,1.0496015548706055
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,2047,0.5539631843566895
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,2047,0.36350560188293457
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,2047,0.32294559478759766
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,2047,0.31358239650726316
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,2047,0.30899200439453123
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,2047,0.3071696043014526
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,2047,0.30607359409332274
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,2047,2.266484832763672
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,2047,2.0852256774902345
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,2047,2.0602127075195313
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,2047,2.0531375885009764
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,2047,2.0490032196044923
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,2047,2.0476400375366213
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,2047,2.045774459838867
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,512,1,1,4095,1.0176655769348144
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,512,1,2,4095,0.673311996459961
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,512,1,4,4095,0.6144944190979004
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,512,1,8,4095,0.5803952217102051
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,512,1,16,4095,0.5755663871765136
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,512,1,32,4095,0.574179220199585
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,512,1,64,4095,0.5728623867034912
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,512,1,1,4095,4.43604621887207
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,512,1,2,4095,4.097734451293945
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,512,1,4,4095,4.064334487915039
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,512,1,8,4095,4.049955368041992
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,512,1,16,4095,4.043521499633789
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,512,1,32,4095,4.041755294799804
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,512,1,64,4095,4.042708969116211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,1,0.15588959455490112
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,1,0.0856719970703125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,1,0.07166879773139953
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,1,0.0638159990310669
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,1,0.06192960143089295
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,1,0.061159998178482056
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,1,0.06128479838371277
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,1,0.16386560201644898
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,1,0.09639840126037598
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,1,0.0830128014087677
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,1,0.07494080066680908
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,1,0.07041440010070801
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,1,0.06938080191612243
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,1,0.06936320066452026
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,3,0.15386559963226318
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,3,0.08734880089759826
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,3,0.07227680087089539
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,3,0.06461439728736877
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,3,0.06145600080490112
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,3,0.06121439933776855
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,3,0.06091840267181396
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,3,0.1673632025718689
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,3,0.10070240497589111
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,3,0.08594080209732055
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,3,0.07985119819641114
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,3,0.07332640290260314
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,3,0.07257440090179443
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,3,0.07188479900360108
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,7,0.15381920337677002
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,7,0.08696960210800171
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,7,0.07160159945487976
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,7,0.06471520066261291
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,7,0.061659198999404904
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,7,0.06158400177955627
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,7,0.060996800661087036
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,7,0.17419519424438476
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,7,0.10790079832077026
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,7,0.09235039949417115
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,7,0.08705120086669922
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,7,0.0800000011920929
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,7,0.07848960161209106
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,7,0.07766720056533813
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,15,0.15786880254745483
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,15,0.08824639916419982
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,15,0.07210559844970703
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,15,0.06585440039634705
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,15,0.061628800630569455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,15,0.061686402559280394
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,15,0.06130719780921936
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,15,0.1911087989807129
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,15,0.12206239700317383
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,15,0.1068336009979248
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,15,0.10140479803085327
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,15,0.09744160175323487
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,15,0.09423360228538513
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,15,0.09170399904251099
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,31,0.16011840105056763
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,31,0.09057120084762574
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,31,0.07443680167198181
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,31,0.06868320107460021
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,31,0.06314560174942016
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,31,0.06215360164642334
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,31,0.06125119924545288
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,31,0.222326397895813
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,31,0.15205600261688232
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,31,0.1380895972251892
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,31,0.13024799823760985
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,31,0.12685760259628295
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,31,0.12559520006179808
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,31,0.1247488021850586
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,63,0.1637168049812317
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,63,0.09822559952735901
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,63,0.07979360222816467
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,63,0.07335360050201416
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,63,0.07053920030593872
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,63,0.06913120150566102
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,63,0.06894720196723939
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,63,0.27904961109161375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,63,0.21485118865966796
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,63,0.1979647994041443
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,63,0.19097280502319336
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,63,0.1871600031852722
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,63,0.18625760078430176
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,63,0.18548320531845092
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,127,0.1874608039855957
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,127,0.11592479944229125
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,127,0.09863359928131103
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,127,0.09122880101203919
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,127,0.08722559809684753
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,127,0.08509600162506104
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,127,0.08445760011672973
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,127,0.4117455959320068
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,127,0.341756796836853
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,127,0.3241888046264648
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,127,0.3159679889678955
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,127,0.31178081035614014
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,127,0.3093600034713745
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,127,0.3088367938995361
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,255,0.23882720470428467
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,255,0.1504032015800476
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,255,0.13314080238342285
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,255,0.12554399967193602
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,255,0.12310400009155273
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,255,0.12054879665374756
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,255,0.12027360200881958
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,255,0.6794400215148926
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,255,0.5888576030731201
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,255,0.5733839988708496
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,255,0.5678127765655517
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,255,0.5638127803802491
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,255,0.5618639945983886
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,255,0.5605279922485351
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,511,0.40071520805358884
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,511,0.252291202545166
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,511,0.2218031883239746
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,511,0.2069711923599243
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,511,0.19954240322113037
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,511,0.19695520401000977
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,511,0.19605280160903932
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,511,1.2719103813171386
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,511,1.1268655776977539
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,511,1.0957632064819336
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,511,1.0817647933959962
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,511,1.0740431785583495
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,511,1.0725839614868165
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,511,1.0705696105957032
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,1023,0.6260335922241211
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,1023,0.4099936008453369
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,1023,0.35966079235076903
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,1023,0.34391040802001954
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,1023,0.33514719009399413
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,1023,0.3319760084152222
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,1023,0.32956318855285643
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,1023,2.340932846069336
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,1023,2.1345247268676757
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,1023,2.0969568252563477
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,1023,2.0840175628662108
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,1023,2.075035285949707
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,1023,2.070283126831055
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,1023,2.0694751739501953
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,128,1024,1,1,2047,1.0977840423583984
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,64,1024,1,2,2047,0.7204224109649658
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,32,1024,1,4,2047,0.649457597732544
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,16,1024,1,8,2047,0.6119359970092774
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,8,1024,1,16,2047,0.6020480155944824
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,4,1024,1,32,2047,0.5983903884887696
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,float16,2,1024,1,64,2047,0.5961040019989013
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,128,1024,1,1,2047,4.5309406280517575
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,64,1024,1,2,2047,4.149262237548828
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,32,1024,1,4,2047,4.102633666992188
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,16,1024,1,8,2047,4.07762565612793
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,8,1024,1,16,2047,4.0687103271484375
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,4,1024,1,32,2047,4.0655567169189455
SGLang,0.5.10,NVIDIA H200,mla_generation,flash_attention,float16,fp8,2,1024,1,64,2047,4.061692810058593
