framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,1,1,0,0.012921600043773651
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,1,4,0,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,1,8,0,0.012204799801111221
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,1,16,0,0.012193600088357926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,1,32,0,0.012211199849843979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,1,64,0,0.012060800194740295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,1,1,0,0.01863359957933426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,1,2,0,0.018190400302410127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,1,4,0,0.018137599527835845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,1,2,0,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,1,8,0,0.01797440052032471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,1,16,0,0.01783200055360794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,1,32,0,0.017910400032997133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,1,64,0,0.01785759925842285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,16,1,0,0.013998399674892425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,16,2,0,0.013977600634098053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,16,4,0,0.013896000385284425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,16,64,0,0.012788799405097962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,16,1,0,0.020084799826145174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,16,8,0,0.01982239931821823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,16,16,0,0.01934400051832199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,16,32,0,0.018771199882030486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,16,64,0,0.01873600035905838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,32,1,0,0.0151296004652977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,32,4,0,0.013851200044155122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,32,8,0,0.013731199502944946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,32,16,0,0.013831999897956849
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,32,32,0,0.013228799402713775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,32,64,0,0.012972800433635712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,32,1,0,0.02128639966249466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,32,2,0,0.02011999934911728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,32,4,0,0.01992799937725067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,32,8,0,0.019996799528598785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,32,16,0,0.01956319957971573
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,32,32,0,0.019363200664520262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,32,64,0,0.0188400000333786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,64,1,0,0.015756799280643462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,64,2,0,0.01520639955997467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,64,4,0,0.015143999457359314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,64,8,0,0.013790400326251983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,64,16,0,0.013873599469661713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,64,32,0,0.013804799318313599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,64,64,0,0.013574400544166565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,64,1,0,0.022393600642681123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,64,2,0,0.021513600647449494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,64,4,0,0.02019679993391037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,64,8,0,0.020244799554347992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,64,16,0,0.01988160014152527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,64,32,0,0.020073600113391876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,64,64,0,0.019487999379634857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,128,1,0,0.017983999848365784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,128,2,0,0.017003199458122252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,128,4,0,0.016596800088882445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,128,8,0,0.015651200711727143
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,128,16,0,0.01563519984483719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,128,32,0,0.015423999726772308
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,128,1,0,0.02449280023574829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,128,64,0,0.014614400267601014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,128,2,0,0.023364800214767455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,128,4,0,0.02343519926071167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,128,8,0,0.022708800435066224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,128,16,0,0.021956799924373625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,128,32,0,0.021510399878025055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,128,64,0,0.020720000565052032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,256,1,0,0.031332799792289735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,256,2,0,0.020588800311088562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,256,4,0,0.019817599654197694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,256,8,0,0.019070400297641753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,256,16,0,0.018907199800014495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,256,32,0,0.018804800510406495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,256,1,0,0.04071359932422638
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,256,64,0,0.017657600343227386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,256,2,0,0.02779200077056885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,256,4,0,0.026582399010658266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,256,8,0,0.026371198892593383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,256,16,0,0.025143998861312866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,256,32,0,0.025220799446105956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,256,64,0,0.0236175999045372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,512,1,0,0.07068960070610046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,512,2,0,0.03827680051326752
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,512,4,0,0.027395200729370118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,512,8,0,0.03557440042495728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,512,16,0,0.028246399760246278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,512,32,0,0.0239439994096756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,512,64,0,0.022864000499248506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,512,1,0,0.08046560287475586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,512,2,0,0.04778240025043488
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,512,4,0,0.03421440124511719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,512,8,0,0.045582398772239685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,512,16,0,0.03548159897327423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,512,32,0,0.031083199381828307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,512,64,0,0.029912000894546507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,1024,1,0,0.1788591980934143
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,1024,2,0,0.09569919705390931
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,1024,4,0,0.0518992006778717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,1024,8,0,0.040940800309181215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,1024,16,0,0.04381439983844757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,1024,32,0,0.03213919997215271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,1024,64,0,0.027659198641777037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,1024,2,0,0.10503360033035278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,1024,1,0,0.1878607988357544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,1024,4,0,0.0620576024055481
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,1024,8,0,0.04813919961452484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,1024,16,0,0.05392799973487854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,1024,32,0,0.04139040112495422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,1024,64,0,0.03503200113773346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,1536,1,0,0.3320336103439331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,1536,2,0,0.1744320034980774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,1536,4,0,0.09606720209121704
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,1536,8,0,0.05507519841194153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,1536,16,0,0.06157119870185852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,1536,32,0,0.042788800597190854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,1536,64,0,0.03420639932155609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,1536,1,0,0.3407376050949097
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,1536,2,0,0.18430240154266359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,1536,4,0,0.10685280561447144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,1536,8,0,0.06439359784126282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,1536,16,0,0.07288159728050232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,1536,32,0,0.054731202125549314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,1536,64,0,0.04411199986934662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,2048,1,0,0.5358096122741699
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,2048,2,0,0.2784912109375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,2048,4,0,0.14508479833602905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,2048,8,0,0.07936639785766601
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,2048,16,0,0.06877760291099548
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,2048,32,0,0.05966240167617798
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,2048,64,0,0.04163680076599121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,2048,1,0,0.5453663825988769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,2048,2,0,0.28877279758453367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,2048,4,0,0.1566815972328186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,2048,8,0,0.09171040058135986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,2048,16,0,0.07901279926300049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,2048,32,0,0.0724943995475769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,2048,64,0,0.05299839973449707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,16,8,0,0.013840000331401824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,16,32,0,0.012936000525951386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,16,16,0,0.013388800621032714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,16,2,0,0.01991039961576462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,32,2,0,0.014180800318717957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,16,4,0,0.019862399995326997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,3072,16,0,0.09789760112762451
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,3072,8,0,0.16251519918441773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,3072,64,0,0.06129119992256164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,3072,2,0,0.5546688079833985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,3072,4,0,0.28588480949401857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,3072,32,0,0.08317599892616272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,3072,1,0,1.0710672378540038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,3072,8,0,0.17846399545669556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,3072,16,0,0.11129920482635498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,3072,64,0,0.07603039741516113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,3072,32,0,0.0973151981830597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,3072,4,0,0.29777119159698484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,3072,2,0,0.5630767822265625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,3072,1,0,1.0818320274353028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,4096,8,0,0.24947359561920165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,4096,16,0,0.1349135994911194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,4096,32,0,0.12416319847106934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,4096,4,0,0.4775231838226318
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,4096,64,0,0.09113919734954834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,4096,2,0,0.9426032066345215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,4096,16,0,0.14965280294418334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,4096,8,0,0.26290719509124755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,4096,32,0,0.13863680362701417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,4096,64,0,0.1079327940940857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,4096,4,0,0.48528480529785156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,4096,2,0,0.9335391998291016
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,6144,16,0,0.2861520051956177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,6144,8,0,0.5211472034454345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,4096,1,0,1.8098976135253906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,6144,32,0,0.19084639549255372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,6144,64,0,0.1294384002685547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,6144,4,0,0.9944128036499024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,4096,1,0,1.8350351333618165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,6144,8,0,0.5153728008270264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,6144,32,0,0.21287360191345214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,6144,16,0,0.33056321144104006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,6144,4,0,1.0097439765930176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,6144,64,0,0.14782079458236694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,6144,2,0,2.045568084716797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,8192,16,0,0.5009471893310546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,8192,8,0,0.8602784156799317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,6144,2,0,2.099787139892578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,8192,32,0,0.2697871923446655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,8192,64,0,0.23399360179901124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,8192,4,0,1.8230688095092773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,6144,1,0,4.964553451538086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,6144,1,0,4.08117904663086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,8192,8,0,1.0101471900939942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,8192,4,0,1.714731216430664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,8192,32,0,0.28560800552368165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,8192,64,0,0.2868079900741577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,8192,16,0,0.5570591926574707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,8192,2,0,3.7433982849121095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,10240,16,0,0.7542736053466796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,8192,2,0,3.6149486541748046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,10240,32,0,0.4194159984588623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,10240,64,0,0.3184880018234253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,10240,8,0,2.5811519622802734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,10240,4,0,2.8000288009643555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,8192,1,0,7.569439697265625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,10240,8,0,1.476857566833496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,8192,1,0,7.063294219970703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,10240,16,0,0.7550079822540283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,10240,32,0,0.44831361770629885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,10240,2,0,5.783295822143555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,10240,64,0,0.328273606300354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,10240,4,0,4.78540153503418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,10240,2,0,5.563425445556641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,12288,16,0,0.9776880264282226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,12288,32,0,0.571614408493042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,12288,4,0,4.142630386352539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,12288,64,0,0.36198399066925047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,10240,1,0,11.205824279785157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,12288,8,0,3.748948669433594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,10240,1,0,11.239482879638672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,12288,16,0,1.9490848541259767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,12288,8,0,1.9461408615112306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,12288,2,0,8.273442840576172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,12288,4,0,3.877462387084961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,12288,64,0,0.40642399787902833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,12288,32,0,0.5692912101745605
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,16384,16,0,1.8218719482421875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,12288,2,0,7.97149887084961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,16384,32,0,1.020796775817871
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,16384,64,0,0.46587200164794923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,16384,8,0,6.681140899658203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,12288,1,0,16.378642272949218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,16384,4,0,7.261879730224609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,16384,16,0,1.7342832565307618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,16384,32,0,0.9225472450256348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,12288,1,0,15.8428955078125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,16384,8,0,3.4606464385986326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,16384,64,0,0.5182640075683593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,16384,2,0,15.320248413085938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,16384,4,0,11.529910278320312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,16384,2,0,14.308859252929688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,32768,32,0,3.716836929321289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,1,32768,64,0,1.8559888839721679
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,16384,1,0,29.744418334960937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,32768,8,0,15.092329406738282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,32768,16,0,12.944488525390625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,16384,1,0,28.172341918945314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,32768,16,0,7.503355407714844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,32768,4,0,30.4192138671875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,1,32768,64,0,1.8842304229736329
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,1,1,0,0.013550400733947754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,1,2,0,0.013158400356769562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,1,4,0,0.012995199859142303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,1,8,0,0.01281919926404953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,1,16,0,0.012699200212955475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,32768,32,0,3.6829471588134766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,1,32,0,0.012852799892425538
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,1,1,0,0.01936960071325302
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,1,2,0,0.018795199692249298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,1,64,0,0.013335999846458436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,1,4,0,0.018651199340820313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,1,8,0,0.019246399402618408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,1,16,0,0.017847999930381775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,1,32,0,0.01929759979248047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,1,64,0,0.01778720021247864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,16,1,0,0.016523200273513793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,16,2,0,0.014273600280284881
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,16,8,0,0.013844799995422364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,16,4,0,0.015432000160217285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,16,16,0,0.013326400518417358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,16,32,0,0.013006399571895599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,16,64,0,0.012582400441169738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,16,1,0,0.02147040069103241
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,16,2,0,0.01964160054922104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,16,4,0,0.019808000326156615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,16,8,0,0.019515199959278105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,16,16,0,0.019483199715614317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,16,32,0,0.01852640062570572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,16,64,0,0.01855839937925339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,32,1,0,0.01528639942407608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,32,2,0,0.015300799906253815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,32,4,0,0.013862399756908417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,32,8,0,0.014044800400733947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,32,16,0,0.013566400110721587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,32,32,0,0.013443200290203095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,32,64,0,0.012720000743865967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,32,1,0,0.02157920002937317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,32,2,0,0.021055999398231506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,32,4,0,0.019940799474716185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,32,8,0,0.019679999351501463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,32,16,0,0.019681599736213685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,32,32,0,0.019017599523067474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,32,64,0,0.01891999989748001
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,64,1,0,0.016047999262809753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,64,4,0,0.015158399939537048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,64,2,0,0.01642560064792633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,64,8,0,0.01515520066022873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,64,16,0,0.014112000167369843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,64,64,0,0.01366879940032959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,64,32,0,0.014129599928855896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,64,1,0,0.023230400681495667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,64,2,0,0.022100800275802614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,64,4,0,0.021675199270248413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,64,8,0,0.021929599344730377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,64,16,0,0.020300799608230592
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,64,32,0,0.020286400616168977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,64,64,0,0.01974560022354126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,128,1,0,0.031497600674629214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,128,2,0,0.018119999766349794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,128,4,0,0.017528000473976135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,128,8,0,0.0168272003531456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,128,16,0,0.016022400557994844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,128,32,0,0.01592639982700348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,128,64,0,0.015004800260066986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,128,1,0,0.037380799651145935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,128,2,0,0.02465600073337555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,128,4,0,0.023678399622440338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,128,8,0,0.02324160039424896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,32768,4,0,28.919134521484374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,128,16,0,0.022049599885940553
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,128,32,0,0.02300640046596527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,128,64,0,0.020824000239372253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,32768,8,0,29.699380493164064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,256,2,0,0.03219519853591919
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,256,1,0,0.05835999846458435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,256,8,0,0.0206496000289917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,256,16,0,0.020846399664878845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,256,32,0,0.019814400374889372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,256,64,0,0.019244800508022308
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,256,4,0,0.04814240038394928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,256,2,0,0.04129279851913452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,256,1,0,0.06729440093040466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,256,4,0,0.06327040195465088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,256,16,0,0.027241599559783936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,256,8,0,0.027956798672676086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,256,32,0,0.03915199935436249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,256,64,0,0.02476480007171631
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,512,1,0,0.1267135977745056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,512,4,0,0.039427199959754945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,512,2,0,0.10000319480895996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,512,8,0,0.03346560001373291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,512,64,0,0.025255998969078063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,512,16,0,0.03733119964599609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,512,32,0,0.039017599821090695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,512,1,0,0.1362607955932617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,512,2,0,0.08093119859695434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,512,8,0,0.04104639887809754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,512,4,0,0.059139198064804076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,512,16,0,0.045844799280166625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,512,32,0,0.03771359920501709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,512,64,0,0.039587199687957764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,1024,2,0,0.17584480047225953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,1024,4,0,0.1149791955947876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,1024,1,0,0.33774240016937257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,1024,8,0,0.05395200252532959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,1024,16,0,0.06016960144042969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,1024,32,0,0.0474128007888794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,1024,64,0,0.0346560001373291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,1024,4,0,0.10794399976730347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,1024,2,0,0.18597760200500488
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,1024,8,0,0.06752480268478393
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,1024,32,0,0.059227198362350464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,1024,16,0,0.059038400650024414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,1024,1,0,0.41849122047424314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,1024,64,0,0.047009599208831784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,1536,8,0,0.09869120121002198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,1536,4,0,0.174235200881958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,1536,16,0,0.06701120138168334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,1536,2,0,0.379967999458313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,1536,32,0,0.07392479777336121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,1536,64,0,0.04877919852733612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,1536,1,0,0.6404607772827149
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,1536,4,0,0.18875839710235595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,1536,8,0,0.11165119409561157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,1536,2,0,0.34467999935150145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,1536,16,0,0.08191840052604675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,1536,32,0,0.08670719861984252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,1536,64,0,0.06323999762535096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,1536,1,0,1.0062288284301757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,2048,4,0,0.2766063928604126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,2048,2,0,0.5362912178039551
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,2048,16,0,0.08451200127601624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,2048,8,0,0.23328640460968017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,2048,32,0,0.0784608006477356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,2048,64,0,0.06578879952430725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,2048,1,0,1.0549983978271484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,2048,4,0,0.2881455898284912
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,2048,8,0,0.17361119985580445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,2048,16,0,0.09960640072822571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,2048,2,0,0.5434639930725098
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,2048,32,0,0.1001471996307373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,2048,64,0,0.08246240019798279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,3072,4,0,0.5521200180053711
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,2048,1,0,1.7650335311889649
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,3072,8,0,0.2907344102859497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,3072,2,0,1.1219840049743652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,3072,32,0,0.11595040559768677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,3072,64,0,0.10776159763336182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,3072,16,0,0.30715041160583495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,3072,1,0,2.181497573852539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,3072,4,0,0.956668758392334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,3072,8,0,0.35240960121154785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,3072,2,0,1.1340527534484863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,3072,32,0,0.1526352047920227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,3072,64,0,0.128057599067688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,3072,16,0,0.30216159820556643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,3072,1,0,2.1385183334350586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,32768,2,0,58.52064208984375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,4096,4,0,0.9575471878051758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,4096,8,0,0.5176047801971435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,4096,32,0,0.1526576042175293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,4096,16,0,0.28626399040222167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,4096,64,0,0.14948639869689942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,4096,1,0,3.863137435913086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,4096,2,0,4.133721542358399
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,4096,2,0,1.8644319534301759
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,4096,16,0,0.2894943952560425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,4096,32,0,0.17555680274963378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,4096,4,0,1.063422393798828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,4096,64,0,0.17453919649124144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,4096,8,0,1.0743871688842774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,4096,1,0,3.8848239898681642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,6144,8,0,1.0868000030517577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,6144,16,0,0.5341728210449219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,6144,64,0,0.2179248094558716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,6144,32,0,0.33411359786987305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,6144,2,0,4.354508972167968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,6144,4,0,4.557572937011718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,6144,4,0,2.116092872619629
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,6144,8,0,1.964998435974121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,6144,2,0,4.214596939086914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,6144,64,0,0.2628432035446167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,6144,16,0,0.5852943897247315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,6144,32,0,0.6264400005340576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,6144,1,0,8.817848205566406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,32768,2,0,55.309539794921875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,8192,8,0,1.8237104415893555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,6144,1,0,8.268476867675782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,8192,64,0,0.2820591926574707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,8192,32,0,0.5332335948944091
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,8192,16,0,0.9346112251281739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,8192,2,0,7.607548522949219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,8192,4,0,8.156934356689453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,8192,4,0,3.6970062255859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,8192,32,0,0.5543024063110351
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,8192,64,0,0.3192624092102051
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,8192,8,0,1.9347375869750976
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,8192,16,0,1.795377540588379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,8192,2,0,7.237009429931641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,10240,8,0,3.0171039581298826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,8192,1,0,15.014303588867188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,10240,32,0,0.8864640235900879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,10240,16,0,1.6922704696655273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,10240,64,0,0.5014095783233643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,8192,1,0,14.34345703125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,10240,2,0,12.157697296142578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,10240,4,0,13.258990478515624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,10240,4,0,5.50425910949707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,10240,8,0,2.8964511871337892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,10240,32,0,0.8324831962585449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,10240,64,0,0.49845280647277834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,10240,16,0,3.0511327743530274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,10240,2,0,11.723131561279297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,10240,1,0,23.179348754882813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,12288,8,0,4.189988708496093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,12288,32,0,1.1248687744140624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,12288,64,0,0.6472527980804443
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,12288,16,0,2.0319839477539063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,10240,1,0,22.62279815673828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,12288,2,0,16.713897705078125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,12288,4,0,20.131275939941407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,32768,1,0,117.60341796875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,12288,8,0,4.067467117309571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,12288,64,0,0.7051599979400635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,12288,4,0,8.106249237060547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,12288,32,0,1.1205056190490723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,12288,16,0,4.612651062011719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,12288,2,0,16.259434509277344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,16384,16,0,3.6731552124023437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,32768,1,0,112.44281005859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,16384,32,0,1.8650991439819335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,16384,64,0,0.9304464340209961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,12288,1,0,34.20328369140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,16384,4,0,15.178134155273437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,12288,1,0,32.965234375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,16384,8,0,18.417218017578126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,16384,8,0,7.2044525146484375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,16384,64,0,1.0125887870788575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,16384,16,0,3.513256072998047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,16384,32,0,4.106687927246094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,16384,4,0,14.691729736328124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,16384,2,0,30.376113891601562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,16384,2,0,28.939508056640626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,32768,16,0,15.0919921875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,32768,32,0,7.628963470458984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,2,32768,64,0,4.073233413696289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,32768,8,0,29.258905029296876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,16384,1,0,59.70478515625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,16384,1,0,56.536199951171874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,32768,16,0,14.305848693847656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,32768,8,0,28.535882568359376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,32768,32,0,7.324089813232422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,1,1,0,0.014214399456977844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,1,2,0,0.01366879940032959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,1,4,0,0.013521599769592284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,2,32768,64,0,3.7141265869140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,1,8,0,0.013406400382518769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,1,16,0,0.0133200004696846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,1,32,0,0.013353599607944489
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,1,64,0,0.013344000279903411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,1,1,0,0.020238399505615234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,1,2,0,0.01990559995174408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,1,4,0,0.01969120055437088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,1,8,0,0.018406400084495546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,1,16,0,0.01831520050764084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,1,32,0,0.018329599499702455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,1,64,0,0.018223999440670012
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,16,1,0,0.016438399255275727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,16,2,0,0.015131199359893798
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,16,4,0,0.013945600390434265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,16,8,0,0.013790400326251983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,16,16,0,0.013396799564361572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,16,32,0,0.013033600151538849
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,16,64,0,0.012675200402736665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,16,1,0,0.02221280038356781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,16,2,0,0.02136639952659607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,16,4,0,0.019991999864578246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,16,8,0,0.019870400428771973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,16,16,0,0.019519999623298645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,16,32,0,0.01900479942560196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,16,64,0,0.018760000169277192
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,32,1,0,0.016249600052833556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,32,2,0,0.015603199601173401
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,32,4,0,0.015254400670528412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,32,8,0,0.013953599333763122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,32,16,0,0.013820800185203552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,32,32,0,0.013467200100421906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,32,64,0,0.012912000715732574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,32,1,0,0.022833600640296936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,32,2,0,0.021987199783325195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,32,4,0,0.021532799303531646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,32,8,0,0.020100800693035124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,32,16,0,0.020070399343967437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,32,32,0,0.019359999895095827
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,32,64,0,0.01897439956665039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,64,1,0,0.028560000658035278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,64,2,0,0.016407999396324157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,64,4,0,0.015603199601173401
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,64,8,0,0.015561600029468537
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,64,16,0,0.014201599359512328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,64,32,0,0.01414559930562973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,64,64,0,0.013857600092887879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,64,1,0,0.03573119938373566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,64,2,0,0.02335519939661026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,64,4,0,0.02215680032968521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,64,8,0,0.0219200000166893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,64,16,0,0.02167679965496063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,64,32,0,0.020339199900627138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,64,64,0,0.019844800233840942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,128,1,0,0.05159519910812378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,128,2,0,0.03152799904346466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,128,4,0,0.018187199532985688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,128,8,0,0.0177824005484581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,128,16,0,0.017566399276256563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,128,64,0,0.015928000211715698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,128,32,0,0.016935999691486358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,128,1,0,0.06186720132827759
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,128,2,0,0.039787200093269345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,128,4,0,0.025201600790023804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,128,8,0,0.024432000517845155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,128,16,0,0.024052800238132478
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,128,32,0,0.023446400463581086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,128,64,0,0.02197760045528412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,256,2,0,0.058899199962615965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,256,1,0,0.10203839540481567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,256,4,0,0.03546079993247986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,256,8,0,0.02272319942712784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,256,16,0,0.022094400227069856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,256,32,0,0.022249600291252135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,256,64,0,0.020895999670028687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,256,1,0,0.11248960494995117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,256,2,0,0.06945440173149109
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,256,4,0,0.046823999285697936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,256,8,0,0.03059839904308319
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,256,16,0,0.029633599519729614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,256,32,0,0.029361599683761598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,256,64,0,0.027585598826408386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,512,1,0,0.23360960483551024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,512,2,0,0.1282528042793274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,512,4,0,0.07124320268630982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,512,8,0,0.05675680041313171
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,512,16,0,0.036646398901939395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,512,32,0,0.04044159948825836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,512,64,0,0.03277280032634735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,512,2,0,0.13958560228347777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,512,1,0,0.24777441024780272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,512,4,0,0.08371359705924988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,512,8,0,0.0699887990951538
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,512,16,0,0.04672960042953491
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,512,32,0,0.05260480046272278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,512,64,0,0.04479039907455444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,1024,2,0,0.3402479887008667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,1024,1,0,0.6525824069976807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,1024,4,0,0.17902719974517822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,1024,16,0,0.07946079969406128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,1024,8,0,0.10120320320129395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,1024,32,0,0.05532479882240295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,1024,64,0,0.054155200719833374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,1024,1,0,0.6662879943847656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,1024,2,0,0.3546639919281006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,1024,4,0,0.1948863983154297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,1024,8,0,0.11686079502105713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,1024,16,0,0.09642720222473145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,1024,32,0,0.07185440063476563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,1024,64,0,0.07167519927024842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,1536,1,0,1.2784576416015625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,1536,2,0,0.6646080017089844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,1536,4,0,0.3418704032897949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,1536,8,0,0.1856816053390503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,1536,16,0,0.12436480522155761
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,1536,32,0,0.09040799736976624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,1536,64,0,0.08709759712219238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,1536,1,0,1.301150417327881
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,1536,4,0,0.35820000171661376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,1536,2,0,0.6770112037658691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,1536,8,0,0.20455679893493653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,1536,16,0,0.14417760372161864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,1536,32,0,0.1113968014717102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,32768,4,0,56.79512939453125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,1536,64,0,0.10676480531692505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,2048,4,0,0.6033567905426025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,2048,2,0,1.0692288398742675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,2048,8,0,0.3059632062911987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,2048,16,0,0.1694704055786133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,2048,32,0,0.1366304039955139
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,2048,64,0,0.09843199849128723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,2048,1,0,2.1337711334228517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,2048,4,0,0.6085631847381592
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,2048,2,0,1.1124032020568848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,2048,8,0,0.3365391969680786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,2048,16,0,0.18899680376052858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,2048,32,0,0.16494239568710328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,2048,64,0,0.12180960178375244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,2048,1,0,2.0922063827514648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,3072,4,0,1.128222370147705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,3072,8,0,0.6176352024078369
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,3072,2,0,2.2464879989624023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,3072,16,0,0.3373215913772583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,3072,32,0,0.22209599018096923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,3072,64,0,0.16231839656829833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,3072,1,0,4.606329727172851
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,3072,2,0,2.273691177368164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,3072,4,0,1.1993984222412108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,3072,8,0,0.6392928123474121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,3072,32,0,0.2479151964187622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,3072,16,0,0.3638943910598755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,3072,64,0,0.19140160083770752
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,3072,1,0,4.419145584106445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,4096,4,0,1.9142383575439452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,4096,8,0,1.0741824150085448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,4096,2,0,3.965091323852539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,4096,16,0,0.5662415981292724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,4096,32,0,0.3155055999755859
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,4096,64,0,0.24485280513763427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,4096,1,0,7.889672088623047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,4096,2,0,3.873982238769531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,4096,4,0,1.8978288650512696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,4096,8,0,1.0460911750793458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,4096,32,0,0.3401263952255249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,4096,16,0,0.5690239906311035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,4096,64,0,0.27735199928283694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,4096,1,0,7.794732666015625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,6144,4,0,4.380644989013672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,32768,2,0,118.5048095703125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,6144,8,0,2.3081008911132814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,6144,2,0,8.604235076904297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,6144,16,0,1.134169578552246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,6144,64,0,0.4097104072570801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,6144,32,0,0.6540272235870361
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,6144,4,0,4.279836654663086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,6144,8,0,2.202947235107422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,6144,16,0,1.1813311576843262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,6144,1,0,18.276478576660157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,6144,2,0,8.622138977050781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,6144,32,0,0.673038387298584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,6144,64,0,0.46851038932800293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,6144,1,0,17.06023254394531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,8192,4,0,7.731005096435547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,8192,16,0,1.9732751846313477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,32768,2,0,114.6437744140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,8192,32,0,1.0170720100402832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,8192,64,0,0.5927807807922363
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,8192,8,0,3.9189056396484374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,8192,2,0,15.63910369873047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,8192,8,0,3.6754417419433594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,8192,4,0,7.5756782531738285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,8192,32,0,1.076103973388672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,8192,16,0,1.9338815689086915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,8192,64,0,0.6229296207427979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,8192,2,0,14.909698486328125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,8192,1,0,30.747073364257812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,32768,4,0,162.96085205078126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,10240,8,0,5.976596832275391
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,10240,32,0,1.6153999328613282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,10240,4,0,12.14343490600586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,10240,64,0,0.8938511848449707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,8192,1,0,29.574871826171876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,10240,16,0,8.360702514648438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,10240,2,0,23.716810607910155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,10240,16,0,3.0036319732666015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,10240,32,0,1.6482303619384766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,10240,8,0,5.742839813232422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,10240,64,0,0.9186767578125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,10240,2,0,22.3428955078125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,10240,1,0,47.766580200195314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,10240,4,0,29.936642456054688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,12288,8,0,8.54837417602539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,12288,32,0,2.2437679290771486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,12288,4,0,17.572833251953124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,12288,64,0,1.2466655731201173
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,12288,16,0,11.49111328125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,10240,1,0,45.732034301757814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,12288,2,0,34.170223999023435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,12288,16,0,4.352390289306641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,12288,32,0,2.2502384185791016
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,12288,8,0,8.331613159179687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,12288,64,0,1.3018336296081543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,32768,1,0,234.649609375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,12288,2,0,32.84513854980469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,32768,1,0,224.8179443359375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,12288,1,0,69.49344482421876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,16384,32,0,4.118369674682617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,12288,4,0,43.407223510742185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,16384,16,0,7.643297576904297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,4,16384,64,0,2.0480159759521483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,16384,8,0,15.18743896484375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,16384,4,0,29.99280700683594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,16384,16,0,7.326519775390625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,16384,8,0,14.614422607421876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,12288,1,0,64.82489013671875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,1,1,0,0.01456640064716339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,1,2,0,0.014075200259685516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,1,4,0,0.013787199556827546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,16384,32,0,3.8271663665771483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,1,8,0,0.014043200016021728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,1,32,0,0.012814399600028992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,1,64,0,0.01279519945383072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,1,16,0,0.013468800485134125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,1,1,0,0.019593599438667297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,1,2,0,0.01918720006942749
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,1,4,0,0.018993599712848662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,1,8,0,0.01889760047197342
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,1,16,0,0.018721599876880646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,1,32,0,0.018785600364208222
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,1,64,0,0.0182096004486084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,4,16384,64,0,2.051041603088379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,16,1,0,0.016920000314712524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,16,2,0,0.015729600191116334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,16,4,0,0.016200000047683717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,16,8,0,0.014812800288200378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,16,16,0,0.013713599741458892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,16,32,0,0.014027200639247894
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,16,64,0,0.01340319961309433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,16,1,0,0.023035199940204622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,16,2,0,0.023499199748039247
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,16,8,0,0.020244799554347992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,16,16,0,0.02091040015220642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,16,4,0,0.02210240066051483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,16,32,0,0.019092799723148347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,16,64,0,0.019739200174808503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,32,1,0,0.029164800047874452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,32,2,0,0.016448000073432924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,32,8,0,0.016337600350379945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,32,4,0,0.016332800686359405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,32,16,0,0.01433439999818802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,32,32,0,0.014206400513648987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,32,64,0,0.014044800400733947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,32,1,0,0.03759520053863526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,32,2,0,0.02356480062007904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,32,8,0,0.022700800001621245
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,32,4,0,0.02354079931974411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,32,16,0,0.02115200012922287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,32,32,0,0.0198512002825737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,32,64,0,0.020214399695396422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,64,1,0,0.04937599897384644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,64,2,0,0.026982399821281432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,64,8,0,0.016708800196647645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,64,4,0,0.01769919991493225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,64,16,0,0.01635199934244156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,64,32,0,0.01576319932937622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,64,64,0,0.014791999757289887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,64,1,0,0.059838402271270755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,64,2,0,0.03684319853782654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,64,8,0,0.023475199937820435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,64,16,0,0.02282080054283142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,64,4,0,0.024201600253582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,64,32,0,0.02136639952659607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,64,64,0,0.020763200521469117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,128,1,0,0.08950880169868469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,128,4,0,0.031148800253868104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,128,2,0,0.0524399995803833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,128,16,0,0.019156800210475923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,128,32,0,0.01918720006942749
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,128,8,0,0.019676800072193145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,128,64,0,0.017956799268722533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,128,1,0,0.09997280240058899
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,128,2,0,0.06312320232391358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,128,4,0,0.041643199324607846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,128,8,0,0.02667039930820465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,128,16,0,0.026180800795555115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,128,32,0,0.026473599672317504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,128,64,0,0.02454719990491867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,256,1,0,0.18537919521331786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,256,2,0,0.10338399410247803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,256,8,0,0.03952800035476685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,256,4,0,0.06044160127639771
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,256,16,0,0.025561600923538208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,256,32,0,0.0256415992975235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,256,64,0,0.024963200092315674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,256,2,0,0.11679199934005738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,256,1,0,0.19683200120925903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,256,4,0,0.07354720234870911
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,256,8,0,0.052934402227401735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,256,16,0,0.03583360016345978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,256,32,0,0.034703999757766724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,256,64,0,0.03420960009098053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,512,2,0,0.24260640144348145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,512,1,0,0.4505631923675537
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,512,4,0,0.1315168023109436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,512,8,0,0.09492319822311401
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,512,16,0,0.06467360258102417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,512,32,0,0.04263199865818024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,512,64,0,0.04905439913272858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,512,2,0,0.25859200954437256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,512,1,0,0.46594557762145994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,512,4,0,0.14763360023498534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,512,8,0,0.11106239557266236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,512,16,0,0.08206400275230408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,512,32,0,0.05883679986000061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,512,64,0,0.06591359972953796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,16384,2,0,60.31825561523438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,1024,2,0,0.6687920093536377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,16384,4,0,28.757733154296876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,1024,8,0,0.19321919679641725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,1024,4,0,0.4005040168762207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,1024,16,0,0.1590127944946289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,1024,32,0,0.09805759787559509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,1024,1,0,1.2791808128356934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,1024,64,0,0.0779807984828949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,1024,8,0,0.24530560970306398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,1024,4,0,0.3665424108505249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,1024,16,0,0.18336000442504882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,1024,2,0,0.6997392177581787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,1024,32,0,0.12037919759750366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,1024,64,0,0.1067728042602539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,1536,8,0,0.36450560092926027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,1024,1,0,1.3683343887329102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,1536,4,0,0.6644207954406738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,1536,16,0,0.2310336112976074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,1536,32,0,0.16091840267181395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,1536,64,0,0.11791839599609374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,1536,2,0,1.315281581878662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,1536,4,0,0.6943823814392089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,1536,8,0,0.38613600730895997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,1536,32,0,0.18766239881515503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,1536,16,0,0.2636336088180542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,1536,2,0,1.3736736297607421
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,1536,64,0,0.14981600046157836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,1536,1,0,2.5613920211791994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,2048,8,0,0.5859407901763916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,1536,1,0,2.5708608627319336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,2048,4,0,1.0851936340332031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,2048,16,0,0.32382879257202146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,2048,64,0,0.1681264042854309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,2048,32,0,0.23274879455566405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,2048,2,0,2.347096061706543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,2048,4,0,1.1490351676940918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,2048,8,0,0.6958784103393555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,2048,16,0,0.36851038932800295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,2048,32,0,0.28481440544128417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,2048,64,0,0.21211841106414794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,2048,2,0,2.1720863342285157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,2048,1,0,4.37035026550293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,3072,8,0,1.2128447532653808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,2048,1,0,4.344851303100586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,3072,4,0,2.2907791137695312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,3072,32,0,0.407257604598999
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,3072,16,0,0.6426127910614013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,3072,64,0,0.28876960277557373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,3072,2,0,4.702228927612305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,3072,4,0,2.3717472076416017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,3072,8,0,1.3326128005981446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,3072,16,0,0.6994400024414062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,3072,64,0,0.3367232084274292
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,3072,32,0,0.4744624137878418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,3072,2,0,4.57073745727539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,3072,1,0,9.437201690673827
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,4096,4,0,3.91851692199707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,4096,8,0,1.9476144790649415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,3072,1,0,8.935334777832031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,4096,64,0,0.4238431930541992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,4096,32,0,0.5811984062194824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,4096,16,0,1.1037440299987793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,4096,2,0,8.110187530517578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,4096,4,0,3.9685009002685545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,4096,8,0,2.0574272155761717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,4096,16,0,1.1149632453918457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,4096,32,0,0.6458047866821289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,4096,64,0,0.4790287971496582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,16384,2,0,57.2869140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,4096,2,0,7.804904174804688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,4096,1,0,15.810731506347656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,6144,16,0,2.20709285736084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,6144,8,0,4.391239929199219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,6144,32,0,1.2937423706054687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,6144,64,0,0.7873936176300049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,4096,1,0,15.837355041503907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,6144,4,0,9.368305969238282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,6144,8,0,4.476176071166992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,6144,16,0,2.3105295181274412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,6144,4,0,9.074508666992188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,6144,2,0,17.918092346191408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,6144,32,0,1.2554384231567384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,6144,64,0,0.8465663909912109
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,6144,2,0,16.539874267578124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,16384,1,0,119.17073974609374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,8192,8,0,7.544609832763672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,8192,16,0,3.965321731567383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,8192,32,0,2.071847915649414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,8192,4,0,15.563258361816406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,8,8192,64,0,1.1278495788574219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,6144,1,0,36.024349975585935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,6144,1,0,33.65200500488281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,8192,8,0,7.574260711669922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,8192,16,0,3.924127960205078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,8192,32,0,2.1060768127441407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,8,8192,64,0,1.2572879791259766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,1,1,0,0.014424000680446625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,1,2,0,0.013713599741458892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,1,4,0,0.013500800728797913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,1,8,0,0.013470399379730224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,1,16,0,0.0134320005774498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,1,32,0,0.013446399569511413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,1,64,0,0.013355199992656708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,1,1,0,0.02035519927740097
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,1,2,0,0.019790400564670563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,1,4,0,0.019623999297618867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,1,8,0,0.019439999759197236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,1,16,0,0.018488000333309173
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,1,32,0,0.01974399983882904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,1,64,0,0.018692800402641298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,16,1,0,0.029624000191688538
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,16,2,0,0.016950400173664094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,16,4,0,0.016358399391174318
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,16,8,0,0.016249600052833556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,16,16,0,0.015412800014019012
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,16,64,0,0.013995200395584106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,16,32,0,0.013974399864673614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,16,1,0,0.03634240031242371
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,16,2,0,0.02348800003528595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,16,4,0,0.022673599421977997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,16,8,0,0.022281600534915923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,16,16,0,0.021033599972724915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,16,32,0,0.020423999428749083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,16,64,0,0.020257599651813507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,32,1,0,0.049825599789619444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,32,2,0,0.02739199995994568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,32,4,0,0.01727039963006973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,32,8,0,0.016732800006866454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,32,16,0,0.016318400204181672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,32,32,0,0.01525759994983673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,32,64,0,0.01451680064201355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,32,1,0,0.060177600383758544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,32,2,0,0.03660959899425507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,32,4,0,0.024134400486946105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,32,8,0,0.023472000658512116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,32,16,0,0.02282879948616028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,32,32,0,0.021142399311065672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,32,64,0,0.02089280039072037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,8192,2,0,30.81334533691406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,64,1,0,0.08547520041465759
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,64,2,0,0.05116479992866516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,64,8,0,0.018580800294876097
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,64,16,0,0.017995199561119078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,64,4,0,0.03621279895305633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,64,32,0,0.017822399735450745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,64,64,0,0.01652639955282211
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,8192,4,0,15.014227294921875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,64,2,0,0.061027199029922485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,64,4,0,0.0402895987033844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,64,1,0,0.10430560111999512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,64,16,0,0.025067201256752013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,64,32,0,0.024724799394607543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,64,8,0,0.02844479978084564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,64,64,0,0.026631999015808105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,128,4,0,0.05727199912071228
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,128,2,0,0.09052799940109253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,128,1,0,0.16119519472122193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,128,16,0,0.02337760031223297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,128,8,0,0.0384768009185791
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,128,32,0,0.02250719964504242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,128,64,0,0.022073599696159362
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,128,2,0,0.1051408052444458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,128,4,0,0.06794080138206482
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,128,8,0,0.04809440076351166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,128,1,0,0.18205920457839966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,128,16,0,0.03361760079860687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,128,32,0,0.03190720081329346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,128,64,0,0.03270559906959534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,256,2,0,0.19405920505523683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,256,4,0,0.10683519840240478
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,256,1,0,0.3546544075012207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,256,8,0,0.06787840127944947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,256,16,0,0.04698080122470856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,256,32,0,0.03212479948997497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,256,64,0,0.031692799925804135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,256,4,0,0.12244800329208375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,256,2,0,0.21059200763702393
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,256,1,0,0.3760416030883789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,256,8,0,0.08430560231208802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,256,16,0,0.06356160044670105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,256,32,0,0.04784159958362579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,256,64,0,0.04415520131587982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,512,4,0,0.25396640300750734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,512,2,0,0.47435359954833983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,512,8,0,0.17333920001983644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,512,16,0,0.10970720052719116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,512,32,0,0.07905920147895813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,512,1,0,0.8859711647033691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,512,64,0,0.058115202188491824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,512,4,0,0.27349119186401366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,512,2,0,0.4948736190795898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,512,8,0,0.19485599994659425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,512,16,0,0.13327360153198242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,512,1,0,0.9099632263183594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,512,32,0,0.102457594871521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,512,64,0,0.08291040062904358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,1024,4,0,0.6721744060516357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,1024,8,0,0.36338560581207274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,1024,2,0,1.318892765045166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,1024,16,0,0.24818079471588134
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,1024,32,0,0.1666000008583069
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,1024,64,0,0.12535359859466552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,1024,1,0,2.6124336242675783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,1024,2,0,1.3758463859558105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,1024,8,0,0.4097856044769287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,1024,4,0,0.7553040027618408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,1024,32,0,0.20985279083251954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,1024,16,0,0.29452478885650635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,1024,64,0,0.1711951971054077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,1024,1,0,2.592401695251465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,1536,4,0,1.3164671897888183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,1536,8,0,0.7124944210052491
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,1536,32,0,0.2836384057998657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,1536,16,0,0.4505887985229492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,1536,64,0,0.20296480655670165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,1536,2,0,2.6197328567504883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,1536,4,0,1.4129776000976562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,1536,8,0,0.7623199939727783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,1536,2,0,2.6590431213378904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,1536,1,0,5.382038497924805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,8192,2,0,30.446063232421874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,1536,32,0,0.3498703956604004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,1536,16,0,0.4999983787536621
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,1536,64,0,0.2687727928161621
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,2048,8,0,1.1552816390991212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,1536,1,0,5.280369567871094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,2048,16,0,0.6263008117675781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,2048,32,0,0.40865278244018555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,2048,4,0,2.1665119171142577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,2048,64,0,0.2900480031967163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,2048,2,0,4.474671936035156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,2048,4,0,2.2187023162841797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,2048,16,0,0.7162320137023925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,2048,8,0,1.2450624465942384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,2048,32,0,0.5055439949035645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,2048,64,0,0.37776479721069334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,2048,2,0,4.410641479492187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,2048,1,0,9.208261108398437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,2048,1,0,8.766677093505859
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,3072,8,0,2.36090087890625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,3072,4,0,4.6789600372314455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,3072,64,0,0.532097578048706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,3072,16,0,1.2149935722351075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,3072,32,0,0.8009072303771972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,8192,1,0,61.503265380859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,3072,2,0,9.587251281738281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,3072,8,0,2.3882976531982423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,3072,4,0,4.740579223632812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,3072,16,0,1.403163242340088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,3072,32,0,0.8773056030273437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,3072,64,0,0.6405680179595947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,3072,2,0,9.153270721435547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,3072,1,0,18.4396728515625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,4096,8,0,4.273448181152344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,4096,4,0,8.496644592285156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,4096,16,0,2.058340835571289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,4096,32,0,1.1957568168640136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,16,4096,64,0,0.7751359939575195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,8192,1,0,59.21256713867187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,3072,1,0,18.644078063964844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,4096,8,0,4.0355792999267575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,4096,16,0,2.211404800415039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,4096,32,0,1.2685296058654785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,1,1,0,0.014681600034236908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,16,4096,64,0,0.9091423988342285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,4096,2,0,16.476376342773438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,1,2,0,0.01379680037498474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,1,8,0,0.013115200400352477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,1,16,0,0.013179199397563934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,1,32,0,0.013073599338531494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,1,4,0,0.014886400103569031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,32,1,64,0,0.012996800243854523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,1,2,0,0.019067199528217317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,1,1,0,0.022355200350284578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,1,4,0,0.018931199610233308
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,1,8,0,0.020937600731849672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,1,16,0,0.01873600035905838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,1,32,0,0.020667199790477753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,32,1,64,0,0.018716800212860107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,16,1,0,0.051785600185394284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,16,2,0,0.029934400320053102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,16,4,0,0.018415999412536622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,16,8,0,0.01695840060710907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,4096,4,0,8.013513946533203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,16,16,0,0.01674239933490753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,32,16,64,0,0.014878399670124054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,16,32,0,0.016420799493789672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,16,2,0,0.03774240016937256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,16,4,0,0.026043200492858888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,16,1,0,0.060571199655532836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,16,8,0,0.02364480048418045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,16,16,0,0.024145600199699403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,16,32,0,0.021985599398612977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,32,16,64,0,0.021081599593162536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,32,1,0,0.09014400243759155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,32,4,0,0.029654398560523987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,32,8,0,0.020022399723529816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,32,2,0,0.05231680274009705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,32,16,0,0.018328000605106354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,32,32,0,0.018595199286937713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,32,32,64,0,0.016990399360656737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,32,1,0,0.09700160026550293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,32,2,0,0.06415200233459473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,32,4,0,0.04119200110435486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,32,8,0,0.027377599477767946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,32,16,0,0.025140801072120668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,32,32,0,0.0249551996588707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,32,32,64,0,0.025140801072120668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,64,1,0,0.15188159942626953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,64,4,0,0.0565392017364502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,64,2,0,0.0900496006011963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,64,16,0,0.02258560061454773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,64,8,0,0.03293919861316681
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,64,32,0,0.02189279943704605
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,32,64,64,0,0.02168319970369339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,64,1,0,0.16525440216064452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,64,4,0,0.06739040017127991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,64,2,0,0.10221439599990845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,64,16,0,0.03227039873600006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,64,8,0,0.04705120027065277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,64,32,0,0.031172800064086913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,32,64,64,0,0.03096800148487091
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,128,4,0,0.0969103991985321
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,128,2,0,0.16548320055007934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,128,1,0,0.30907840728759767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,128,8,0,0.06288800239562989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,128,16,0,0.04226239919662476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,128,32,0,0.02948639988899231
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,32,128,64,0,0.029041600227355958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,128,4,0,0.11384479999542237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,128,2,0,0.18115359544754028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,128,1,0,0.325761604309082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,128,8,0,0.0787056028842926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,128,16,0,0.05930240154266357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,32,128,64,0,0.04151679873466492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,128,32,0,0.04352000057697296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,256,4,0,0.1971343994140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,256,2,0,0.3757200002670288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,256,8,0,0.1194991946220398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,256,16,0,0.08174719810485839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,256,32,0,0.06030880212783814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,256,1,0,0.6953455924987793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,32,256,64,0,0.045956799387931825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,256,4,0,0.22173600196838378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,256,2,0,0.40460958480834963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,256,8,0,0.14363839626312255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,256,16,0,0.10608799457550049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,256,32,0,0.08597279787063598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,256,1,0,0.7312640190124512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,32,256,64,0,0.0680191993713379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,512,8,0,0.3258447885513306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,512,4,0,0.49766879081726073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,512,32,0,0.1398576021194458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,512,16,0,0.20157439708709718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,512,2,0,0.9560319900512695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,32,512,64,0,0.1076416015625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,512,1,0,1.7615776062011719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,512,4,0,0.5382080078125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,4096,2,0,15.8448974609375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,512,2,0,0.9953120231628418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,512,16,0,0.24627840518951416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,512,8,0,0.37269279956817625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,512,32,0,0.18873759508132934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,32,512,64,0,0.1501968026161194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,512,1,0,1.7886127471923827
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,1024,4,0,1.341217613220215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,1024,8,0,0.7103472232818604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,1024,16,0,0.4748688220977783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,1024,32,0,0.30650880336761477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,32,1024,64,0,0.22233920097351073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,1024,2,0,2.6917152404785156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,1024,4,0,1.4732975959777832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,1024,8,0,0.799616003036499
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,1024,2,0,2.6938720703125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,1024,1,0,5.269059371948242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,4096,1,0,33.01963806152344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,1024,32,0,0.3979408025741577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,1024,16,0,0.5578415870666504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,32,1024,64,0,0.3145296096801758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,1024,1,0,5.220449447631836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,1536,8,0,1.4176783561706543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,1536,32,0,0.5277440071105957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,1536,4,0,2.6697168350219727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,1536,16,0,0.8536128044128418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,32,1536,64,0,0.3646111965179443
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,1536,8,0,1.48123197555542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,1536,2,0,5.440816116333008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,1536,4,0,2.6952592849731447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,1536,16,0,0.962224006652832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,32,1536,64,0,0.4776288032531738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,1536,32,0,0.6578671932220459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,4096,1,0,31.207171630859374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,1536,2,0,5.348092651367187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,2048,16,0,1.2059040069580078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,2048,8,0,2.2843791961669924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,2048,32,0,0.778172779083252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,1536,1,0,11.087850952148438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,2048,4,0,4.480155181884766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,32,2048,64,0,0.5452400207519531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,1536,1,0,10.492552185058594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,2048,8,0,2.398847961425781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,2048,16,0,1.3620431900024415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,2048,2,0,8.994110107421875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,2048,32,0,0.9498687744140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,1,1,0,0.016595199704170227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,1,2,0,0.014524799585342408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,2048,4,0,4.549614334106446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,1,8,0,0.014480000734329224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,1,4,0,0.014913600683212281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,32,2048,64,0,0.6729648113250732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,1,32,0,0.014494399726390838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,64,1,64,0,0.014804799854755402
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,1,1,0,0.022247999906539917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,1,4,0,0.021110400557518005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,1,16,0,0.014819200336933135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,1,8,0,0.02011200040578842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,1,2,0,0.020732800662517547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,1,16,0,0.019926400482654573
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,1,32,0,0.01995680034160614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,64,1,64,0,0.020193600654602052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,16,2,0,0.053192001581192014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,16,1,0,0.08906720280647278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,16,8,0,0.02027679979801178
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,16,4,0,0.0328575998544693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,16,16,0,0.018878400325775146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,64,16,64,0,0.01767839938402176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,16,32,0,0.017833599448204042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,16,1,0,0.09775840044021607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,16,2,0,0.06360639929771424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,16,8,0,0.027699199318885804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,16,4,0,0.043367999792099
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,16,16,0,0.026368001103401185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,16,32,0,0.02492319941520691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,64,16,64,0,0.024987199902534486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,32,1,0,0.15879039764404296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,32,2,0,0.09152640104293823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,32,8,0,0.03557280004024506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,32,4,0,0.05588480234146118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,32,16,0,0.02330400049686432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,32,32,0,0.022152000665664674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,64,32,64,0,0.020822399854660036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,32,1,0,0.16876319646835328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,32,8,0,0.04783200025558472
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,32,4,0,0.06884959936141968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,32,2,0,0.10358239412307739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,32,32,0,0.031446400284767154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,32,16,0,0.03303999900817871
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,64,32,64,0,0.030516800284385682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,64,8,0,0.06273279786109924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,64,4,0,0.09832640290260315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,64,2,0,0.16145440340042114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,64,1,0,0.29769918918609617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,64,16,0,0.04245119988918304
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,64,32,0,0.02969599962234497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,64,64,64,0,0.02874560058116913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,64,4,0,0.11255040168762206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,64,8,0,0.07840160131454468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,64,2,0,0.17973439693450927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,64,1,0,0.31222560405731203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,64,16,0,0.0581279993057251
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,64,32,0,0.04330880045890808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,64,64,64,0,0.04281280040740967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,128,8,0,0.11320799589157104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,128,4,0,0.1801408052444458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,128,2,0,0.3151535987854004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,128,16,0,0.07683039903640747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,128,32,0,0.05623840093612671
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,64,128,64,0,0.04395360052585602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,128,1,0,0.6118383884429932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,128,4,0,0.20623040199279785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,128,2,0,0.3404975891113281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,128,8,0,0.13796000480651854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,128,16,0,0.10080480575561523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,128,1,0,0.6300159931182862
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,2048,2,0,9.096497344970704
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,128,32,0,0.08191199898719788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,64,128,64,0,0.06644160151481629
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,256,8,0,0.22625279426574707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,256,4,0,0.38806560039520266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,256,16,0,0.14902399778366088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,256,32,0,0.11166239976882934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,64,256,64,0,0.08312960267066956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,256,2,0,0.7625648021697998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,256,1,0,1.3762207984924317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,256,2,0,0.7869728088378907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,256,4,0,0.42807998657226565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,256,8,0,0.2704576015472412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,256,16,0,0.1915552020072937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,256,32,0,0.1520975947380066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,64,256,64,0,0.12370719909667968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,256,1,0,1.4440095901489258
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,2048,1,0,17.79004669189453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,512,8,0,0.6435264110565185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,512,4,0,0.9708432197570801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,64,512,64,0,0.19437919855117797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,512,32,0,0.258025598526001
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,512,16,0,0.40133280754089357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,512,2,0,1.8612176895141601
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,512,16,0,0.45746397972106934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,512,8,0,0.7103792190551758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,512,4,0,1.0467167854309083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,64,512,64,0,0.27409279346466064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,512,32,0,0.33252480030059817
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,512,1,0,3.5680606842041014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,512,2,0,1.969108772277832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,1024,16,0,0.927790355682373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,1024,8,0,1.4096480369567872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,512,1,0,3.612527847290039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,2048,1,0,17.9729248046875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,1024,32,0,0.5849103927612305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,64,1024,64,0,0.4143712043762207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,1024,4,0,2.6529375076293946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,1024,8,0,1.5475279808044433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,1024,16,0,1.0937824249267578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,64,1024,64,0,0.5710256099700928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,1024,32,0,0.759503984451294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,1024,4,0,2.840407943725586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,1,1,0,0.01814880073070526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,1024,2,0,5.482968139648437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,1,2,0,0.01658879965543747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,1,4,0,0.015643200278282164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,1,32,0,0.015694400668144225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,1,16,0,0.015883199870586395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,128,1,64,0,0.015545600652694702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,1,8,0,0.015774400532245637
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,1,1,0,0.02401760071516037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,1,2,0,0.022023999691009523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,1,4,0,0.021238400042057036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,1,16,0,0.02163040041923523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,1,32,0,0.02183839976787567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,1,8,0,0.021481600403785706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,128,1,64,0,0.021201600134372712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,16,4,0,0.05598400235176086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,16,8,0,0.03636159896850586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,16,2,0,0.09216639995574952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,16,1,0,0.1588752031326294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,16,16,0,0.023230400681495667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,16,32,0,0.02199999988079071
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,128,16,64,0,0.021960000693798064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,16,1,0,0.1688639998435974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,16,4,0,0.06976799964904785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,16,8,0,0.048588800430297854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,16,2,0,0.10327520370483398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,16,16,0,0.03398720026016235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,16,32,0,0.03159840106964111
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,128,16,64,0,0.03195840120315552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,32,2,0,0.16449439525604248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,32,8,0,0.06403040289878845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,32,4,0,0.09827039837837219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,32,1,0,0.2930383920669556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,32,16,0,0.043331199884414674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,32,32,0,0.02993279993534088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,128,32,64,0,0.02887200117111206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,32,8,0,0.0803600013256073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,32,4,0,0.11371040344238281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,32,1,0,0.31058878898620607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,32,2,0,0.17986719608306884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,32,16,0,0.06019999980926514
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,32,32,0,0.04382559955120087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,128,32,64,0,0.04132960140705109
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,64,8,0,0.11164480447769165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,64,4,0,0.17640000581741333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,64,2,0,0.30750079154968263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,64,16,0,0.075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,1024,2,0,5.841513442993164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,64,32,0,0.054889601469039914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,64,1,0,0.590990400314331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,128,64,64,0,0.04415040016174317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,64,8,0,0.13567839860916137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,64,4,0,0.20182878971099855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,64,2,0,0.3315887928009033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,64,16,0,0.10030239820480347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,64,32,0,0.08069120049476623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,128,64,64,0,0.06915839910507202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,64,1,0,0.5989744186401367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,128,8,0,0.20913760662078856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,128,4,0,0.3448751926422119
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,128,16,0,0.1406432032585144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,128,32,0,0.10532480478286743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,128,2,0,0.6182496070861816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,128,128,64,0,0.079448002576828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,128,1,0,1.1982831954956055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,128,4,0,0.38605918884277346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,1024,1,0,11.220275115966796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,128,2,0,0.6604623794555664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,128,16,0,0.18251680135726928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,128,8,0,0.2514240026473999
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,128,128,64,0,0.12023520469665527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,128,32,0,0.16111359596252442
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,128,1,0,1.2411279678344727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,256,16,0,0.27962241172790525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,256,8,0,0.4555503845214844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,256,4,0,0.749073600769043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,256,32,0,0.2028656005859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,1024,1,0,10.424881744384766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,128,256,64,0,0.15599679946899414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,256,2,0,1.4916447639465331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,256,16,0,0.3623663902282715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,256,8,0,0.5243807792663574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,256,4,0,0.8335712432861329
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,256,32,0,0.28507840633392334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,256,1,0,2.768142318725586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,256,2,0,1.5710880279541015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,128,256,64,0,0.2389888048171997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,512,16,0,0.7310304164886474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,256,1,0,2.8465904235839843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,512,8,0,1.255465602874756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,512,32,0,0.48440961837768554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,512,4,0,1.9286544799804688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,128,512,64,0,0.3627935886383057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,512,2,0,3.771625518798828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,512,16,0,0.8991488456726074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,512,8,0,1.407686424255371
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,1,1,0,0.030950400233268737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,1,2,0,0.02338559925556183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,512,4,0,2.0804943084716796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,128,512,64,0,0.516428804397583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,1,4,0,0.02282080054283142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,512,32,0,0.6479904174804687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,1,8,0,0.02172800004482269
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,1,32,0,0.022356800734996796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,1,16,0,0.02253919988870621
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,256,1,64,0,0.02234079986810684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,1,2,0,0.029142400622367857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,1,1,0,0.036820799112319946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,1,4,0,0.02842719852924347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,1,8,0,0.029014399647712706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,1,16,0,0.028420799970626832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,1,32,0,0.028265601396560668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,256,1,64,0,0.028265601396560668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,16,8,0,0.06405599713325501
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,16,4,0,0.09961119890213013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,16,2,0,0.16748640537261963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,16,1,0,0.29680159091949465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,16,16,0,0.04304479956626892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,16,32,0,0.03538239896297455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,256,16,64,0,0.03452000021934509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,512,2,0,3.913278579711914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,16,8,0,0.07978559732437134
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,16,4,0,0.1152575969696045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,16,2,0,0.17940160036087036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,16,1,0,0.3144576072692871
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,16,32,0,0.04937280118465424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,256,16,64,0,0.047884801030159
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,16,16,0,0.06268159747123718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,32,16,0,0.07676799893379212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,32,8,0,0.11245280504226685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,32,2,0,0.31027679443359374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,32,4,0,0.17868640422821044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,32,32,0,0.05504480004310608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,256,32,64,0,0.04908320009708404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,32,1,0,0.5809775829315186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,512,1,0,7.03056640625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,32,8,0,0.13606079816818237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,32,2,0,0.3326879978179932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,32,4,0,0.20199360847473144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,32,16,0,0.10058879852294922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,32,1,0,0.6031375885009765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,256,32,64,0,0.0746944010257721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,32,32,0,0.08202239871025085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,64,4,0,0.34370079040527346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,64,16,0,0.14230560064315795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,64,8,0,0.21082398891448975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,64,32,0,0.10539840459823609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,64,2,0,0.6091695785522461
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,256,64,64,0,0.08281279802322387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,64,8,0,0.25075199604034426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,64,4,0,0.3837519884109497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,64,1,0,1.132472038269043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,64,2,0,0.6585504055023194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,512,1,0,7.111215972900391
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,64,16,0,0.18471200466156007
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,64,32,0,0.14916479587554932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,256,64,64,0,0.12397119998931885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,64,1,0,1.182652759552002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,128,16,0,0.26307680606842043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,128,8,0,0.4029520034790039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,128,32,0,0.1954383969306946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,128,4,0,0.6706048011779785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,256,128,64,0,0.1519503951072693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,128,2,0,1.2305168151855468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,128,8,0,0.47869601249694826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,128,4,0,0.750161600112915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,128,32,0,0.2755232095718384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,128,16,0,0.3449840068817139
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,128,2,0,1.2950143814086914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,256,128,64,0,0.2321552038192749
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,128,1,0,2.3819135665893554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,256,16,0,0.5465167999267578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,256,8,0,0.8481103897094726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,128,1,0,2.462499237060547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,256,32,0,0.3848367929458618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,1,256,256,64,0,0.2934479951858521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,256,4,0,1.4772319793701172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,256,8,0,1.0089983940124512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,256,2,0,2.9783552169799803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,256,4,0,1.6392688751220703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,256,16,0,0.6971712112426758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,1,1,0,0.013337600231170654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,1,2,0,0.012880000472068786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,256,32,0,0.5465295791625977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,1,4,0,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,1,256,256,64,0,0.4509439945220947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,1,16,0,0.012068799883127212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,1,8,0,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,1,32,0,0.012171199917793274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,1,64,0,0.012108799815177918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,1,1,0,0.01913439929485321
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,1,2,0,0.018760000169277192
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,1,4,0,0.018355199694633485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,1,8,0,0.018166400492191315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,1,16,0,0.01808159947395325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,1,32,0,0.018059200048446654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,1,64,0,0.018035200238227845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,16,1,0,0.015387199819087982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,16,2,0,0.015091200172901154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,16,4,0,0.014071999490261078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,16,8,0,0.013972799479961395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,16,16,0,0.013607999682426453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,256,2,0,3.0812416076660156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,16,32,0,0.013556799292564392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,16,64,0,0.01313920021057129
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,16,1,0,0.021329599618911742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,16,8,0,0.019964799284934998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,16,4,0,0.02001439929008484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,16,16,0,0.0197952002286911
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,16,32,0,0.019571200013160706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,16,2,0,0.019942399859428406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,16,64,0,0.019257600605487823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,32,1,0,0.015166400372982025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,32,2,0,0.01552480012178421
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,32,8,0,0.013889600336551667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,32,16,0,0.013716800510883332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,32,32,0,0.01377280056476593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,32,4,0,0.015193599462509155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,32,64,0,0.013495999574661254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,32,1,0,0.0220768004655838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,256,1,0,5.492102432250976
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,32,2,0,0.0213359996676445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,32,4,0,0.020052799582481386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,32,8,0,0.019684800505638124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,32,16,0,0.019678400456905366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,32,64,0,0.01915840059518814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,64,1,0,0.016233600676059723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,64,4,0,0.015137599408626556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,64,2,0,0.016017599403858183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,32,32,0,0.019339199364185333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,64,8,0,0.01510400027036667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,64,16,0,0.013996799290180207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,64,32,0,0.013924799859523773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,64,64,0,0.01361439973115921
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,64,2,0,0.022177599370479584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,64,4,0,0.021447999775409697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,64,16,0,0.019939200580120088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,64,8,0,0.019947199523448943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,64,1,0,0.022673599421977997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,64,32,0,0.019937600195407867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,64,64,0,0.019968000054359437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,128,1,0,0.030435198545455934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,128,2,0,0.0174575999379158
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,128,8,0,0.016732800006866454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,128,16,0,0.01566559970378876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,128,32,0,0.015518400073051452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,128,4,0,0.016814400255680085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,128,64,0,0.015329599380493164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,128,1,0,0.03964959979057312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,128,4,0,0.023375999927520753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,128,2,0,0.023824000358581544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,128,16,0,0.021534399688243867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,128,32,0,0.021726399660110474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,128,8,0,0.022257600724697114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,128,64,0,0.021663999557495116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,256,1,0,0.058595198392868045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,256,4,0,0.020904000103473663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,256,2,0,0.03351680040359497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,256,16,0,0.019324800372123717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,256,8,0,0.019620800018310548
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,256,32,0,0.01957920044660568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,256,64,0,0.018681600689888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,256,1,0,0.06811519861221313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,256,2,0,0.04344319999217987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,256,8,0,0.02645759880542755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,256,16,0,0.025444799661636354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,256,4,0,0.02689119875431061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,256,32,0,0.025035199522972108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,256,64,0,0.02534720003604889
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,256,1,0,5.601300811767578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,512,2,0,0.07027999758720398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,512,8,0,0.026974400877952574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,512,1,0,0.12690240144729614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,512,4,0,0.03924480080604553
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,512,16,0,0.03651840090751648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,512,64,0,0.02402399927377701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,512,2,0,0.08074560165405273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,512,8,0,0.03434399962425232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,512,32,0,0.02784479856491089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,512,1,0,0.13644640445709227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,512,16,0,0.045286399126052854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,512,32,0,0.03574880063533783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,512,4,0,0.04798080027103424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,512,64,0,0.031115201115608216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,1024,8,0,0.05154719948768616
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,1024,4,0,0.09555040001869201
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,1024,32,0,0.043863999843597415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,1024,16,0,0.040171200037002565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,1024,2,0,0.17719360589981079
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,1024,64,0,0.0324431985616684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,1024,8,0,0.061900800466537474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,1024,16,0,0.0484607994556427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,1024,1,0,0.32567360401153567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,1024,2,0,0.18631360530853272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,1024,4,0,0.10377119779586792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,1024,1,0,0.3400928020477295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,1024,32,0,0.05267040133476257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,1024,64,0,0.040827199816703796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,1536,8,0,0.09641759991645812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,1536,16,0,0.05580959916114807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,1536,32,0,0.06175199747085571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,1536,64,0,0.04309599995613098
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,1536,2,0,0.3329360008239746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,1536,4,0,0.1725759983062744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,1536,16,0,0.06473280191421509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,1536,8,0,0.10646560192108154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,1536,4,0,0.18289599418640137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,1536,1,0,0.6158559799194336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,1536,2,0,0.3468751907348633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,1536,64,0,0.055060797929763795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,1536,32,0,0.0709119975566864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,1536,1,0,0.6371071815490723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,2048,16,0,0.07708160281181335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,2048,8,0,0.1442512035369873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,2048,4,0,0.2781008005142212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,2048,64,0,0.05798239707946777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,2048,32,0,0.06898080110549927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,2048,2,0,0.5264463901519776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,2048,8,0,0.15767199993133546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,2048,4,0,0.28190720081329346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,2048,16,0,0.08957440257072449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,2048,32,0,0.07900959849357606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,2048,1,0,1.0144880294799805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,2048,64,0,0.07045599818229675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,2048,2,0,0.542846393585205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,3072,16,0,0.1623568058013916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,2048,1,0,1.0116016387939453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,3072,8,0,0.28035359382629393
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,3072,32,0,0.09833599925041199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,3072,64,0,0.08603360056877137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,3072,4,0,0.5813183784484863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,3072,8,0,0.3104576110839844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,3072,2,0,1.0870160102844237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,3072,4,0,0.5858751773834229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,3072,16,0,0.18146879673004152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,3072,32,0,0.11812319755554199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,3072,64,0,0.09971839785575867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,3072,2,0,1.083348846435547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,3072,1,0,2.128116798400879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,4096,8,0,0.46898398399353025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,4096,16,0,0.27221601009368895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,3072,1,0,2.0668655395507813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,4096,4,0,0.9549504280090332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,4096,32,0,0.1344815969467163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,4096,64,0,0.13983360528945923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,4096,16,0,0.28192639350891113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,4096,8,0,0.4857776165008545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,4096,2,0,1.9189104080200194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,4096,32,0,0.1545408010482788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,4096,4,0,0.9299471855163575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,4096,64,0,0.15249919891357422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,4096,2,0,1.8856847763061524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,4096,1,0,3.959588623046875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,6144,8,0,1.022798442840576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,6144,4,0,2.1960304260253904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,6144,16,0,0.5500847816467285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,6144,32,0,0.30459520816802976
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,6144,64,0,0.20529119968414306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,4096,1,0,3.8277057647705077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,6144,8,0,1.0705967903137208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,6144,2,0,4.227487945556641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,6144,16,0,0.5531792163848877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,6144,32,0,0.34892001152038576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,6144,4,0,2.0959375381469725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,6144,64,0,0.2163424015045166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,6144,2,0,4.1741584777832035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,6144,1,0,8.73917465209961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,8192,8,0,1.849355125427246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,8192,16,0,0.8722463607788086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,8192,4,0,3.747118377685547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,8192,64,0,0.27002720832824706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,8192,32,0,0.5083759784698486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,6144,1,0,8.204841613769531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,8192,8,0,1.8170560836791991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,8192,32,0,0.5032000064849853
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,8192,16,0,0.9545743942260743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,8192,64,0,0.2889120101928711
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,8192,2,0,7.4741981506347654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,8192,4,0,3.6578929901123045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,10240,8,0,2.779000091552734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,8192,2,0,7.159185791015625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,10240,32,0,0.7336415767669677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,10240,16,0,1.4513888359069824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,10240,64,0,0.4051360130310059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,10240,4,0,5.728585433959961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,8192,1,0,14.773776245117187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,10240,8,0,2.7298736572265625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,8192,1,0,14.115109252929688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,10240,4,0,5.541955184936524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,10240,32,0,0.7697231769561768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,10240,2,0,12.072787475585937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,10240,16,0,1.3947600364685058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,10240,64,0,0.44898557662963867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,12288,8,0,4.135091018676758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,10240,2,0,11.165833282470704
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,12288,32,0,1.0038960456848145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,12288,64,0,0.5751344203948975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,12288,16,0,2.334236717224121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,12288,4,0,9.7943603515625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,10240,1,0,24.982205200195313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,10240,1,0,22.995811462402344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,12288,8,0,4.166329574584961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,12288,2,0,17.400297546386717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,12288,4,0,8.082505798339843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,12288,32,0,1.028825569152832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,12288,64,0,0.6265903949737549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,12288,16,0,1.9730255126953125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,12288,2,0,16.172308349609374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,16384,8,0,7.467951965332031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,16384,16,0,3.5874046325683593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,16384,32,0,1.8403072357177734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,16384,64,0,0.9414015769958496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,16384,4,0,15.011915588378907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,12288,1,0,34.348297119140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,12288,1,0,32.29119873046875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,16384,8,0,6.9066162109375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,16384,32,0,1.7276639938354492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,16384,16,0,3.5188655853271484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,16384,64,0,0.9094351768493653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,16384,1,0,317.7858642578125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,16384,4,0,14.711820983886719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,16384,2,0,30.168695068359376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,16384,2,0,29.365032958984376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,1,32768,16,0,14.829620361328125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,1,32768,32,0,7.326590728759766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,1,32768,64,0,3.6562862396240234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,16384,1,0,59.43784790039062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,1,32768,8,0,29.828900146484376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,16384,1,0,57.21568603515625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,1,32768,16,0,14.187828063964844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,1,32768,8,0,28.62308349609375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,1,32768,32,0,7.0664924621582035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,1,1,0,0.014776000380516052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,1,2,0,0.014100800454616546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,1,4,0,0.013716800510883332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,1,8,0,0.01340319961309433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,1,16,0,0.013310399651527405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,1,32,0,0.012380799651145935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,1,64,0,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,1,1,0,0.019499200582504272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,1,2,0,0.018873600661754607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,1,4,0,0.01841759979724884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,1,8,0,0.018199999630451203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,1,16,0,0.01820800006389618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,1,32,0,0.017636799812316896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,1,64,0,0.0177279993891716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,16,1,0,0.015545600652694702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,1,32768,64,0,3.6475456237792967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,16,2,0,0.017075200378894807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,16,4,0,0.016407999396324157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,16,8,0,0.015526400506496429
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,16,16,0,0.015390400588512421
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,16,32,0,0.015024000406265258
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,16,64,0,0.01465120017528534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,16,1,0,0.022724799811840057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,16,2,0,0.021902400255203246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,16,4,0,0.020977599918842314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,16,8,0,0.02051679939031601
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,16,16,0,0.02041600048542023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,16,32,0,0.019704000651836397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,16,64,0,0.019196799397468566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,32,1,0,0.01664319932460785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,32,2,0,0.01566080003976822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,32,4,0,0.015548799932003022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,32,8,0,0.014337599277496338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,32,16,0,0.01435520052909851
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,32,32,0,0.014150400459766389
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,32,64,0,0.013795199990272521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,32,1,0,0.023528000712394713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,32,2,0,0.02276480048894882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,32,4,0,0.021823999285697938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,32,8,0,0.020694400370121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,32,16,0,0.02056799978017807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,32,32,0,0.02046239972114563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,32,64,0,0.020019200444221497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,64,1,0,0.028841599822044373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,64,2,0,0.016646400094032288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,64,4,0,0.015441599488258361
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,64,8,0,0.015535999834537507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,64,16,0,0.015455999970436096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,64,32,0,0.014046399295330048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,64,64,0,0.013953599333763122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,64,1,0,0.03313600122928619
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,64,2,0,0.022998400032520294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,64,4,0,0.021766400337219237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,64,8,0,0.021451200544834136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,64,16,0,0.020015999674797058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,64,32,0,0.020046399533748628
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,64,64,0,0.019827200472354888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,128,1,0,0.05140479803085327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,128,2,0,0.029633599519729614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,128,4,0,0.017560000717639922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,128,8,0,0.016915200650691985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,128,16,0,0.01668799966573715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,128,32,0,0.016774399578571318
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,128,64,0,0.01564960032701492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,128,1,0,0.061475199460983274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,128,2,0,0.035920000076293944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,128,4,0,0.024321599304676055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,128,8,0,0.023824000358581544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,128,16,0,0.02305919975042343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,128,32,0,0.022891199588775633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,128,64,0,0.021848000586032867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,256,1,0,0.10070079565048218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,256,2,0,0.05731840133666992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,256,4,0,0.031780800223350524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,256,8,0,0.02125120013952255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,256,16,0,0.02048639953136444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,256,32,0,0.020342400670051573
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,256,64,0,0.01971520036458969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,256,2,0,0.06688640117645264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,256,1,0,0.11183359622955322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,256,4,0,0.04044800102710724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,256,8,0,0.02815679907798767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,256,16,0,0.027422401309013366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,256,32,0,0.02727839946746826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,256,64,0,0.026105600595474242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,512,1,0,0.2312623977661133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,512,2,0,0.12656160593032836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,512,8,0,0.03949280083179474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,512,4,0,0.07040479779243469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,512,16,0,0.03366079926490784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,512,32,0,0.03742879927158356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,512,64,0,0.029872000217437744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,512,1,0,0.24129118919372558
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,512,2,0,0.1369935989379883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,512,8,0,0.04959999918937683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,512,4,0,0.08055520057678223
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,512,16,0,0.041073599457740785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,512,32,0,0.04750880002975464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,512,64,0,0.03733760118484497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,1024,2,0,0.3349296092987061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,1024,1,0,0.6429759979248046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,1024,4,0,0.17696640491485596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,1024,8,0,0.09702720046043396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,1024,16,0,0.05397599935531616
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,1024,32,0,0.04921759963035584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,1024,64,0,0.04669440090656281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,1024,2,0,0.3449408054351807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,1024,1,0,0.6607600212097168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,1024,4,0,0.188919997215271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,1024,16,0,0.06710559725761414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,1024,8,0,0.10902080535888672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,1024,32,0,0.05939040184020996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,1024,64,0,0.0606719970703125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,1536,2,0,0.6569952011108399
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,1536,4,0,0.3389296054840088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,1536,1,0,1.2278351783752441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,1536,8,0,0.17585920095443724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,1536,16,0,0.09995200037956238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,1536,32,0,0.06736000180244446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,1536,64,0,0.07448800206184387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,1536,1,0,1.2402400016784667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,1536,2,0,0.6622000217437745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,1536,4,0,0.34927520751953123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,1536,8,0,0.18926399946212769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,1536,16,0,0.11379040479660034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,1536,32,0,0.0819920003414154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,1536,64,0,0.08891680240631103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,2048,2,0,1.069155216217041
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,2048,4,0,0.547926378250122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,2048,1,0,2.0099552154541014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,2048,8,0,0.28119840621948244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,2048,16,0,0.16468800306320192
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,2048,32,0,0.08551200032234192
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,2048,64,0,0.08544480204582214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,2048,2,0,1.1009743690490723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,2048,4,0,0.5659759998321533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,2048,1,0,2.061350440979004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,2048,8,0,0.2960959911346436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,2048,16,0,0.17141439914703369
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,2048,32,0,0.10144959688186646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,2048,64,0,0.09976159930229186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,1,32768,4,0,56.37870483398437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,3072,2,0,2.2442079544067384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,3072,4,0,1.140329647064209
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,3072,16,0,0.30866079330444335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,3072,8,0,0.6127088069915771
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,3072,32,0,0.17926239967346191
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,3072,64,0,0.12422399520874024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,3072,1,0,4.633607864379883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,3072,2,0,2.241441535949707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,3072,4,0,1.222535991668701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,3072,16,0,0.3342655897140503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,3072,8,0,0.591871976852417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,3072,32,0,0.1991328001022339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,3072,64,0,0.139956796169281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,3072,1,0,4.3617408752441404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,4096,4,0,1.9471599578857421
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,4096,2,0,3.9474464416503907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,4096,8,0,1.0153087615966796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,4096,32,0,0.28048319816589357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,4096,16,0,0.5313104152679443
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,4096,64,0,0.15222079753875734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,4096,1,0,7.958380889892578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,4096,2,0,3.7712047576904295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,4096,8,0,0.9789648056030273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,4096,4,0,1.915292739868164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,4096,32,0,0.2919919967651367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,4096,16,0,0.5256927967071533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,4096,64,0,0.1766432046890259
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,4096,1,0,7.389291381835937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,6144,4,0,4.2699951171875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,6144,8,0,2.117291259765625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,6144,2,0,8.97426528930664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,6144,16,0,1.092252826690674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,6144,32,0,0.6974368095397949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,6144,64,0,0.33489279747009276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,1,32768,2,0,116.96995849609375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,6144,4,0,4.153793716430664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,6144,8,0,2.1066240310668944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,6144,1,0,17.822398376464843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,6144,32,0,0.6030128002166748
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,6144,16,0,1.0841279983520509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,6144,64,0,0.354422402381897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,6144,2,0,9.089987182617188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,8192,4,0,7.737204742431641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,6144,1,0,17.103001403808594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,8192,16,0,1.873632049560547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,1,32768,2,0,113.70831298828125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,8192,32,0,0.9878239631652832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,8192,8,0,3.7625457763671877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,8192,64,0,0.5264848232269287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,8192,2,0,15.475814819335938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,8192,8,0,3.585126495361328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,8192,4,0,7.559220886230468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,8192,32,0,0.9849519729614258
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,8192,16,0,1.8423999786376952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,8192,64,0,0.5157792091369628
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,8192,2,0,14.7496826171875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,8192,1,0,30.636541748046874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,10240,8,0,5.894582366943359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,10240,4,0,12.166031646728516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,10240,16,0,2.963425636291504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,10240,32,0,1.5272560119628906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,10240,64,0,0.8024191856384277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,8192,1,0,30.061282348632812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,10240,2,0,24.48462677001953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,10240,8,0,5.672143936157227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,10240,4,0,11.42052001953125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,10240,16,0,2.8176591873168944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,10240,32,0,1.4691583633422851
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,10240,64,0,0.874289608001709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,1,32768,4,0,189.54971923828126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,10240,2,0,22.754884338378908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,10240,1,0,47.4244384765625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,12288,8,0,8.434497833251953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,12288,16,0,4.271219253540039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,12288,32,0,2.1569696426391602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,12288,64,0,1.1357104301452636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,10240,1,0,45.409872436523436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,12288,2,0,34.73678283691406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,12288,8,0,8.129291534423828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,12288,4,0,16.258390808105467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,12288,16,0,4.096696090698242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,12288,64,0,1.141312026977539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,12288,32,0,2.0202016830444336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,1,32768,1,0,233.7781005859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,12288,2,0,31.32586669921875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,12288,4,0,54.141632080078125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,1,32768,1,0,227.0923583984375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,12288,1,0,68.53789672851562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,16384,32,0,3.7365791320800783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,16384,64,0,1.8985807418823242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,16384,8,0,16.28851776123047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,16384,4,0,29.855804443359375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,16384,16,0,23.688233947753908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,12288,1,0,64.7257080078125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,16384,32,0,3.5557056427001954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,16384,64,0,1.9038639068603516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,16384,8,0,14.340574645996094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,16384,4,0,28.275302124023437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,16384,16,0,20.999879455566408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,16384,2,0,59.6809326171875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,16384,2,0,58.195556640625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,2,32768,16,0,29.79884033203125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,2,32768,32,0,15.166337585449218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,2,32768,64,0,7.474295806884766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,16384,1,0,120.89234619140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,16384,1,0,115.30865478515625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,2,32768,4,0,118.5713623046875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,2,32768,8,0,57.36612548828125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,2,32768,16,0,27.98101806640625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,2,32768,64,0,7.475281524658203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,1,1,0,0.01531040072441101
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,1,2,0,0.014724799990653991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,1,4,0,0.014262400567531586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,1,8,0,0.01403840035200119
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,1,16,0,0.013920000195503235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,1,32,0,0.012803199887275695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,1,64,0,0.012699200212955475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,1,1,0,0.019916799664497376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,2,32768,32,0,14.521615600585937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,1,2,0,0.01961439996957779
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,1,8,0,0.018863999843597413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,1,4,0,0.020478400588035583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,1,16,0,0.018750399351119995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,1,32,0,0.01967200040817261
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,1,64,0,0.018731200695037843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,16,1,0,0.01741439998149872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,16,2,0,0.015438400208950043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,16,4,0,0.015636800229549407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,16,8,0,0.013980799913406372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,16,16,0,0.014291200041770934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,16,32,0,0.013631999492645264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,16,64,0,0.01337919980287552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,16,1,0,0.023030400276184082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,16,2,0,0.022937600314617158
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,16,4,0,0.021587200462818146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,16,8,0,0.020652799308300017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,16,16,0,0.01961279958486557
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,16,32,0,0.02018879950046539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,16,64,0,0.018905599415302277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,32,1,0,0.029003199934959412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,32,2,0,0.016103999316692354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,32,4,0,0.01635040044784546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,32,8,0,0.015559999644756317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,32,16,0,0.015732799470424653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,32,32,0,0.013790400326251983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,32,64,0,0.013739199936389923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,32,1,0,0.03864159882068634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,32,2,0,0.023100799322128295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,32,4,0,0.022300800681114195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,32,8,0,0.021956799924373625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,32,32,0,0.020107200741767882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,32,16,0,0.01992799937725067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,32,64,0,0.01958879977464676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,64,1,0,0.04887360036373138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,64,2,0,0.026188799738883974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,64,4,0,0.01642719954252243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,64,8,0,0.01552959978580475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,64,16,0,0.01587039977312088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,64,32,0,0.014084799587726593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,64,64,0,0.014083200693130493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,64,1,0,0.0588096022605896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,64,2,0,0.033292800188064575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,64,4,0,0.023043200373649597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,64,8,0,0.02297919988632202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,64,16,0,0.022006399929523468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,64,32,0,0.02175839990377426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,64,64,0,0.02038560062646866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,128,1,0,0.08953440189361572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,128,2,0,0.05143200159072876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,128,4,0,0.029108801484107973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,128,8,0,0.018328000605106354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,128,16,0,0.017641599476337432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,128,32,0,0.017417599260807038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,128,64,0,0.016791999340057373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,128,1,0,0.09978880286216736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,128,2,0,0.06198400259017944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,128,4,0,0.03899520039558411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,128,8,0,0.024881599843502043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,128,16,0,0.024327999353408812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,128,32,0,0.02385759949684143
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,128,64,0,0.02303680032491684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,256,1,0,0.18065600395202636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,256,2,0,0.10053440332412719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,256,4,0,0.05852159857749939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,256,8,0,0.034985598921775815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,256,16,0,0.02240000069141388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,256,32,0,0.02195200026035309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,256,64,0,0.022033600509166716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,256,1,0,0.19066720008850097
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,256,2,0,0.11136480569839477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,256,4,0,0.06927840113639831
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,256,8,0,0.04658080041408539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,256,16,0,0.029955199360847472
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,256,32,0,0.02940160036087036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,256,64,0,0.029044800996780397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,512,1,0,0.4372352123260498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,512,2,0,0.23320479393005372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,512,4,0,0.12661279439926149
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,512,8,0,0.07117760181427002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,512,16,0,0.05575039982795715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,512,32,0,0.03641279935836792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,512,64,0,0.04048640131950378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,512,1,0,0.4593247890472412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,512,2,0,0.25055201053619386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,512,4,0,0.14171359539031983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,512,8,0,0.08382880091667175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,512,16,0,0.07097439765930176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,512,32,0,0.04705759882926941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,512,64,0,0.05443040132522583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,1024,2,0,0.6581456184387207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,1024,4,0,0.34420640468597413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,1024,1,0,1.2650256156921387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,1024,8,0,0.1810431957244873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,1024,16,0,0.10276000499725342
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,1024,32,0,0.0804751992225647
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,1024,64,0,0.05617759823799133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,1024,2,0,0.6819136142730713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,1024,4,0,0.3625119924545288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,1024,1,0,1.2831616401672363
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,1024,8,0,0.19869120121002198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,1024,16,0,0.11867519617080688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,1024,32,0,0.09795359969139099
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,1024,64,0,0.0722495973110199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,1536,2,0,1.28548002243042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,1536,4,0,0.6632912158966064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,1536,1,0,2.563027191162109
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,1536,8,0,0.34150879383087157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,1536,32,0,0.12498240470886231
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,1536,16,0,0.20114879608154296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,1536,64,0,0.09073119759559631
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,1536,2,0,1.3112607955932618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,1536,4,0,0.6809184074401855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,1536,1,0,2.5694320678710936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,1536,8,0,0.3595648050308228
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,1536,16,0,0.2134399890899658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,1536,32,0,0.14469759464263915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,1536,64,0,0.11597440242767335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,2048,2,0,2.2072511672973634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,2048,4,0,1.1395343780517577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,2048,1,0,4.289812850952148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,2048,8,0,0.570198392868042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,2048,32,0,0.16960959434509276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,2048,16,0,0.3177135944366455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,2,32768,4,0,114.253564453125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,2048,64,0,0.13542399406433106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,2048,4,0,1.1149151802062989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,2048,8,0,0.5785280227661133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,2048,16,0,0.3195584058761597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,2048,2,0,2.2133407592773438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,2048,32,0,0.18880159854888917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,2048,64,0,0.15998239517211915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,2048,1,0,4.185260772705078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,3072,4,0,2.328740882873535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,3072,2,0,4.465694427490234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,3072,8,0,1.182755184173584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,3072,16,0,0.668126392364502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,3072,32,0,0.3379647970199585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,3072,64,0,0.24165918827056884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,2,32768,8,0,190.3279541015625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,3072,1,0,9.299734497070313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,3072,8,0,1.1634544372558593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,3072,2,0,4.849278259277344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,3072,16,0,0.6211135864257813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,3072,32,0,0.3620815992355347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,3072,64,0,0.24828479290008545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,3072,4,0,6.306980895996094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,3072,1,0,8.828121948242188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,4096,8,0,1.9154207229614257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,4096,16,0,1.070854377746582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,4096,32,0,0.6008927822113037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,4096,64,0,0.3249631881713867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,4096,2,0,8.131534576416016
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,4096,4,0,12.414011383056641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,4096,1,0,16.394332885742188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,4096,2,0,7.9595695495605465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,4096,8,0,2.022150421142578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,4096,16,0,1.0213536262512206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,4096,4,0,3.990752029418945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,4096,32,0,0.5768335819244385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,4096,64,0,0.36328959465026855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,4096,1,0,15.6805908203125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,2,32768,2,0,234.2296875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,6144,4,0,8.870178985595704
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,6144,16,0,2.1408527374267576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,6144,32,0,1.1390959739685058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,6144,64,0,0.628382396697998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,6144,2,0,18.090223693847655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,6144,8,0,12.414222717285156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,6144,4,0,8.590699005126954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,6144,16,0,2.181862449645996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,6144,2,0,16.865159606933595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,6144,32,0,1.1809679985046386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,6144,64,0,0.6935247898101806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,6144,8,0,10.99908676147461
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,6144,1,0,35.338311767578126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,8192,8,0,7.858004760742188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,6144,1,0,33.956109619140626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,8192,16,0,3.9143310546875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,8192,64,0,1.0435968399047852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,8192,32,0,1.9029743194580078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,2,32768,2,0,225.0340087890625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,8192,2,0,30.600827026367188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,8192,4,0,14.840547180175781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,8192,8,0,7.368164825439453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,8192,16,0,3.7416576385498046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,8192,64,0,1.0755552291870116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,8192,32,0,1.9315824508666992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,8192,2,0,29.680270385742187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,8192,4,0,47.916162109375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,8192,1,0,62.2635498046875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,10240,16,0,6.134151840209961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,10240,32,0,3.0615968704223633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,10240,4,0,23.938381958007813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,10240,64,0,1.620140838623047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,8192,1,0,59.487030029296875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,10240,8,0,38.02523193359375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,10240,2,0,47.404327392578125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,10240,16,0,5.746593475341797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,10240,32,0,2.927342414855957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,10240,64,0,1.6004671096801757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,10240,4,0,23.21528015136719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,10240,8,0,33.43875732421875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,10240,2,0,46.52348937988281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,10240,1,0,94.88809814453126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,12288,8,0,17.377822875976562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,12288,32,0,4.3750255584716795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,12288,16,0,8.584260559082031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,12288,64,0,2.219856071472168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,10240,1,0,90.85396728515624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,12288,2,0,68.79434204101562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,12288,8,0,16.46564178466797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,12288,16,0,8.397444915771484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,12288,4,0,32.746768188476565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,12288,64,0,2.166254425048828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,12288,32,0,4.28306884765625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,2,32768,1,0,469.28955078125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,12288,2,0,66.2920166015625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,12288,4,0,116.8807861328125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,12288,1,0,134.80784912109374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,4,16384,8,0,30.533526611328124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,4,16384,32,0,7.553273773193359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,4,16384,64,0,3.888631820678711
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,2,32768,1,0,455.703662109375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,4,16384,4,0,60.00758666992188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,12288,1,0,130.030029296875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,4,16384,16,0,51.86881713867187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,4,16384,16,0,14.633238220214844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,4,16384,64,0,3.7866352081298826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,1,1,0,0.014798399806022645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,1,2,0,0.01401440054178238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,1,4,0,0.013624000549316406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,1,8,0,0.013425600528717042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,1,16,0,0.013324800133705138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,1,32,0,0.01329600065946579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,1,64,0,0.0133775994181633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,1,1,0,0.02117920070886612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,1,2,0,0.02035360038280487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,1,4,0,0.018830400705337525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,1,8,0,0.018654400110244752
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,1,16,0,0.018492799997329713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,1,32,0,0.018508799374103546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,1,64,0,0.01847199946641922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,16,1,0,0.029145601391792297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,16,2,0,0.016681599617004394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,16,4,0,0.016062399744987486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,16,8,0,0.015662400424480437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,16,16,0,0.01555040031671524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,16,32,0,0.01395840048789978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,16,64,0,0.01348160058259964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,16,1,0,0.03663359880447388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,16,2,0,0.02340960055589676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,16,4,0,0.02292319983243942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,16,8,0,0.0219200000166893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,4,16384,8,0,28.957427978515625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,16,16,0,0.022123199701309205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,16,32,0,0.02073120027780533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,16,64,0,0.02014079988002777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,32,1,0,0.05121440291404724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,32,2,0,0.026287999749183655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,32,4,0,0.017195199429988862
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,32,8,0,0.01674239933490753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,32,16,0,0.016251200437545778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,32,32,0,0.015192000567913056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,32,64,0,0.014864000678062438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,32,1,0,0.061590397357940675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,32,2,0,0.03528479933738708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,32,4,0,0.023240000009536743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,32,8,0,0.022891199588775633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,32,16,0,0.0219200000166893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,32,32,0,0.02173279970884323
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,32,64,0,0.02018879950046539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,64,1,0,0.08428959846496582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,64,2,0,0.049275198578834535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,64,4,0,0.02672800123691559
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,64,8,0,0.017297600209712983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,64,16,0,0.01679680049419403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,64,32,0,0.016337600350379945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,64,64,0,0.015254400670528412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,64,1,0,0.09487040042877197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,64,2,0,0.05958719849586487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,64,4,0,0.03638400137424469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,64,8,0,0.024129599332809448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,64,16,0,0.023342399299144743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,64,32,0,0.02292640060186386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,64,64,0,0.021819199621677398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,128,1,0,0.15893919467926027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,128,2,0,0.08933600187301635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,128,4,0,0.05289919972419739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,128,8,0,0.03144800066947937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,128,16,0,0.019729599356651306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,128,32,0,0.019415999948978423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,128,64,0,0.019431999325752257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,128,1,0,0.1699679970741272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,4,16384,4,0,58.258612060546874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,128,2,0,0.10015840530395508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,128,8,0,0.04269599914550781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,128,4,0,0.0689840018749237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,128,16,0,0.026791998744010927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,128,64,0,0.02640799880027771
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,128,32,0,0.02905920147895813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,256,1,0,0.3434799909591675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,256,2,0,0.20179359912872313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,256,4,0,0.10366400480270385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,256,16,0,0.03952800035476685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,256,8,0,0.062480002641677856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,256,32,0,0.02569119930267334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,256,64,0,0.025627198815345763
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,256,1,0,0.35654239654541015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,256,2,0,0.2000335931777954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,256,4,0,0.11637920141220093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,256,16,0,0.054028797149658206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,256,8,0,0.07438719868659974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,4,16384,32,0,23.364724731445314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,256,32,0,0.03573119938373566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,256,64,0,0.03501920104026794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,512,2,0,0.45982561111450193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,512,4,0,0.24446399211883546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,512,8,0,0.13364800214767455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,512,16,0,0.09572960138320923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,512,32,0,0.0663648009300232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,512,64,0,0.043275201320648195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,512,4,0,0.2601471900939941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,512,2,0,0.46521601676940916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,512,8,0,0.14905760288238526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,512,16,0,0.1106943964958191
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,512,32,0,0.08252639770507812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,512,1,0,0.8907584190368653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,512,64,0,0.056987202167510985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,512,1,0,2.1627344131469726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,1024,4,0,0.6653071880340576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,1024,16,0,0.18936959505081177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,1024,8,0,0.87631196975708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,1024,32,0,0.13902560472488404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,1024,2,0,1.3064352035522462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,1024,64,0,0.2273792028427124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,1024,1,0,2.560766410827637
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,1024,2,0,1.3283535957336425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,1024,4,0,1.173459243774414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,1024,8,0,0.3978208065032959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,1024,16,0,0.21894400119781493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,1024,64,0,0.13018239736557008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,1024,32,0,0.3763472080230713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,1024,1,0,2.5388736724853516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,1536,8,0,0.6853695869445801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,1536,16,0,0.36676959991455077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,1536,32,0,0.23864800930023194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,1536,64,0,0.15927200317382811
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,1536,2,0,2.650815963745117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,1536,4,0,3.3657199859619142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,1536,1,0,5.1866302490234375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,4,16384,2,0,120.88011474609375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,1536,2,0,2.652926445007324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,1536,8,0,0.699129581451416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,1536,32,0,0.2695696115493774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,1536,16,0,0.4208384037017822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,1536,64,0,0.19178400039672852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,1536,4,0,3.4653247833251952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,1536,1,0,5.144537734985351
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,2048,8,0,1.1115103721618653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,2048,16,0,0.568561601638794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,2048,32,0,0.3164400100708008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,2048,64,0,0.22465438842773439
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,2048,2,0,4.452687835693359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,2048,4,0,6.601449584960937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,2048,4,0,2.1385520935058593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,2048,1,0,9.045811462402344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,2048,2,0,4.262231826782227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,2048,16,0,0.5999728202819824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,2048,32,0,0.38485279083251955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,2048,64,0,0.27413918972015383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,2048,8,0,2.933399963378906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,2048,1,0,8.59613265991211
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,3072,16,0,1.1516384124755858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,3072,4,0,4.695849609375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,3072,64,0,0.41874241828918457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,3072,32,0,0.6324079990386963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,3072,8,0,7.203334045410156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,3072,2,0,9.780513763427734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,3072,8,0,2.329191970825195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,3072,16,0,1.2267487525939942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,3072,32,0,0.6763328075408935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,3072,64,0,0.45250558853149414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,3072,2,0,10.100931549072266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,3072,1,0,18.201605224609374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,3072,4,0,13.1606201171875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,3072,1,0,18.291969299316406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,4096,16,0,2.032846450805664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,4096,4,0,7.925103759765625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,4096,64,0,0.602620792388916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,4096,32,0,1.0799807548522948
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,4,16384,2,0,116.28660888671875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,4096,2,0,16.570724487304688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,4096,8,0,13.005960083007812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,4096,8,0,3.9134368896484375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,4096,32,0,1.0808208465576172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,4096,64,0,0.6373760223388671
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,4096,4,0,8.095053100585938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,4096,16,0,5.641505432128906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,4096,2,0,15.695173645019532
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,4096,1,0,32.894082641601564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,6144,8,0,8.879110717773438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,6144,16,0,4.410111999511718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,6144,64,0,1.253007984161377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,6144,32,0,2.293600082397461
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,4096,1,0,31.08792724609375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,6144,2,0,36.05262756347656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,6144,4,0,17.79193115234375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,6144,16,0,4.521766281127929
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,6144,32,0,2.2634464263916017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,6144,8,0,8.578401947021485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,6144,64,0,1.2535375595092773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,6144,2,0,33.7546630859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,6144,4,0,56.72184448242187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,4,16384,1,0,239.784814453125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,6144,1,0,71.4041259765625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,8,8192,32,0,3.9152881622314455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,8,8192,16,0,7.791937255859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,8,8192,64,0,2.0702239990234377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,8,8192,4,0,30.965753173828126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,6144,1,0,68.35625610351562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,8,8192,8,0,15.228558349609376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,4,16384,1,0,227.5537109375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,8,8192,32,0,3.9669185638427735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,8,8192,64,0,2.0772607803344725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,1,1,0,0.01528639942407608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,1,2,0,0.013686400651931763
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,1,4,0,0.013414399325847625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,8,8192,16,0,7.6181884765625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,1,8,0,0.01311040073633194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,1,32,0,0.012830400466918945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,1,16,0,0.013662399351596832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,1,64,0,0.012356799840927125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,1,2,0,0.019036799669265747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,8,8192,8,0,50.03041381835938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,1,1,0,0.02226720005273819
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,1,4,0,0.018603199720382692
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,1,16,0,0.0182096004486084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,1,32,0,0.019894400238990785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,1,64,0,0.01839679926633835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,1,8,0,0.06212959885597229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,8,8192,2,0,61.77156982421875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,16,2,0,0.02884480059146881
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,16,1,0,0.05047839879989624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,16,4,0,0.04512799978256225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,16,16,0,0.015607999265193939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,16,8,0,0.018595199286937713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,16,64,0,0.039241600036621097
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,16,1,0,0.05875200033187866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,16,32,0,0.015363200008869171
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,16,4,0,0.023039999604225158
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,16,16,0,0.061136001348495485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,16,2,0,0.04235840141773224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,16,32,0,0.02000479996204376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,16,8,0,0.023447999358177186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,16,64,0,0.033302399516105655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,32,4,0,0.04427840113639832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,32,1,0,0.09525120258331299
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,32,2,0,0.05064799785614014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,32,8,0,0.027561599016189577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,32,16,0,0.01921759992837906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,32,64,0,0.024412800371646882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,32,32,0,0.01674560010433197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,32,2,0,0.08865919709205627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,32,4,0,0.037590399384498596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,32,1,0,0.10668959617614746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,32,8,0,0.028987199068069458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,32,64,0,0.025979200005531312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,32,16,0,0.023524799942970277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,32,32,0,0.026745599508285523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,64,2,0,0.0856719970703125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,64,1,0,0.1761072039604187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,64,8,0,0.029032000899314882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,64,4,0,0.05388960242271423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,64,16,0,0.022516800463199614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,64,32,0,0.018089599907398224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,64,64,0,0.018772800266742707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,64,2,0,0.09634559750556945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,64,1,0,0.1830064058303833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,64,8,0,0.040078398585319516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,64,16,0,0.030500799417495728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,64,32,0,0.024817599356174468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,64,4,0,0.06361600160598754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,64,64,0,0.02934719920158386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,128,4,0,0.10332640409469604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,8,8192,4,0,29.877764892578124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,128,1,0,0.29913280010223386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,128,2,0,0.1670575976371765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,128,8,0,0.06347360014915467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,128,32,0,0.022856000065803527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,128,64,0,0.023203200101852416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,128,16,0,0.039099198579788205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,128,4,0,0.1044592022895813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,128,2,0,0.17453279495239257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,128,1,0,0.3593935966491699
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,128,8,0,0.07321919798851013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,128,16,0,0.049532800912857056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,128,32,0,0.03244960010051727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,128,64,0,0.03751679956912994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,256,8,0,0.12743200063705445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,256,4,0,0.1940448045730591
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,256,2,0,0.36236960887908937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,256,32,0,0.0460783988237381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,256,16,0,0.08380159735679626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,256,64,0,0.03264159858226776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,256,1,0,0.7251296043395996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,256,4,0,0.21231999397277831
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,256,8,0,0.12299519777297974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,256,2,0,0.4725088119506836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,256,16,0,0.08427680134773255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,256,32,0,0.06411679983139038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,256,1,0,0.6922224044799805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,256,64,0,0.06667680144309998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,512,8,0,0.35301759243011477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,512,4,0,0.48354239463806153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,512,32,0,0.11133919954299927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,512,16,0,0.24953598976135255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,512,2,0,0.8827199935913086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,512,64,0,0.07972800135612487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,512,1,0,1.6978464126586914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,512,4,0,0.5073440074920654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,512,2,0,0.9057456016540527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,512,16,0,0.1988368034362793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,512,8,0,0.27783520221710206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,512,32,0,0.13296639919281006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,512,64,0,0.10421439409255981
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,1024,4,0,1.3327887535095215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,1024,16,0,0.37114720344543456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,512,1,0,3.724824142456055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,1024,8,0,0.6840144157409668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,1024,32,0,0.25353920459747314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,1024,2,0,2.6531919479370116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,1024,64,0,0.3555295944213867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,1024,8,0,1.3762784004211426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,1024,4,0,1.455833625793457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,1024,32,0,0.3032639980316162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,1024,2,0,2.6173471450805663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,1024,1,0,5.346822357177734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,1024,64,0,0.21246399879455566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,1024,16,0,0.8883983612060546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,1024,1,0,5.1422382354736325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,1536,4,0,2.679128074645996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,1536,16,0,0.7209392070770264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,1536,32,0,0.4687952041625977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,1536,8,0,3.663955307006836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,1536,64,0,0.28315200805664065
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,1536,2,0,5.392443084716797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,1536,4,0,2.6447792053222656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,1536,8,0,1.4199456214904784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,1536,16,0,0.7588208198547364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,1536,32,0,0.4895199775695801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,1536,64,0,0.3405920028686523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,1536,1,0,10.495622253417968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,1536,1,0,10.186475372314453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,2048,4,0,4.442049789428711
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,8,8192,2,0,60.14406127929688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,2048,2,0,8.918225860595703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,2048,8,0,2.2365936279296874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,1536,2,0,13.614984130859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,2048,32,0,0.6431871891021729
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,2048,16,0,1.1881792068481445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,2048,64,0,0.47084641456604004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,2048,8,0,2.2838815689086913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,2048,16,0,1.2050640106201171
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,2048,32,0,0.6849408149719238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,2048,64,0,0.49321441650390624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,2048,4,0,4.53551025390625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,2048,1,0,17.76958465576172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,2048,2,0,8.575889587402344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,3072,16,0,2.3416032791137695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,3072,32,0,1.3587663650512696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,3072,8,0,5.223355102539062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,3072,64,0,0.8542703628540039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,3072,4,0,9.776076507568359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,3072,2,0,19.269076538085937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,3072,4,0,9.461103820800782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,3072,16,0,2.4177215576171873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,3072,8,0,4.515980911254883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,3072,32,0,1.4002991676330567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,8,8192,1,0,124.19283447265624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,3072,64,0,0.8570832252502442
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,3072,2,0,18.518833923339844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,16,4096,8,0,8.2215087890625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,3072,1,0,38.153628540039065
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,16,4096,32,0,2.2939071655273438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,16,4096,16,0,4.006439971923828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,16,4096,64,0,1.2433568000793458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,2048,1,0,47.25257568359375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,16,4096,4,0,16.705076599121092
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,3072,1,0,36.4701904296875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,8,8192,1,0,118.1849853515625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,16,4096,16,0,4.199195098876953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,16,4096,8,0,8.067108917236329
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,32,1,1,0,0.0166143998503685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,1,2,0,0.015081599354743958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,1,4,0,0.014135999977588654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,1,8,0,0.014017599821090698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,1,16,0,0.013804799318313599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,16,4096,64,0,1.3281871795654296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,1,32,0,0.013947199285030364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,32,1,1,0,0.022166399657726286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,1,64,0,0.013790400326251983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,1,2,0,0.020374399423599244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,1,4,0,0.020108799636363982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,16,4096,32,0,2.1665632247924806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,1,8,0,0.018966400623321535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,1,16,0,0.019734400510787963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,1,64,0,0.018911999464035035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,1,32,0,0.02019200026988983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,32,16,1,0,0.08762720227241516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,16,2,0,0.05055999755859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,16,8,0,0.018644799292087556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,16,16,0,0.017478400468826295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,16,4,0,0.03182240128517151
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,16,32,0,0.01688639968633652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,16,64,0,0.015406399965286255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,16,4,0,0.03670719861984253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,16,2,0,0.061857599020004275
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,16,8,0,0.025028800964355467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,32,16,1,0,0.09788320064544678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,16,16,0,0.02491839975118637
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,16,32,0,0.023203200101852416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,16,64,0,0.02221920043230057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,32,32,1,0,0.15589599609375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,32,2,0,0.08732479810714722
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,32,8,0,0.02996639907360077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,32,16,0,0.01943040043115616
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,32,4,0,0.05247359871864319
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,32,32,0,0.018723200261592864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,32,64,0,0.01842560023069382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,32,2,0,0.09733920097351074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,32,4,0,0.062244802713394165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,32,8,0,0.0412992000579834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,32,16,0,0.02650879919528961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,32,32,1,0,0.163428795337677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,32,32,0,0.025595200061798096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,32,64,0,0.02484000027179718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,64,4,0,0.08880320191383362
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,64,2,0,0.15186560153961182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,64,8,0,0.054953598976135255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,64,16,0,0.03332160115242004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,64,32,0,0.022147199511528014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,32,64,1,0,0.2888031959533691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,64,64,0,0.022017599642276765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,64,4,0,0.10249919891357422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,32,64,1,0,0.2970976114273071
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,64,2,0,0.1681391954421997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,64,8,0,0.06738719940185547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,64,16,0,0.04754399955272674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,64,64,0,0.03115679919719696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,64,32,0,0.032913601398468016
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,128,2,0,0.30950241088867186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,128,4,0,0.16722400188446046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,128,8,0,0.09772480130195618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,32,128,1,0,0.5848127841949463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,128,16,0,0.06347200274467468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,128,32,0,0.04315359890460968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,128,64,0,0.029679998755455017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,128,2,0,0.3260528087615967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,32,128,1,0,0.600052785873413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,128,4,0,0.18197280168533325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,128,8,0,0.11382880210876464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,128,16,0,0.07957760095596314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,128,32,0,0.05970079898834228
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,128,64,0,0.04358560144901276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,16,4096,2,0,33.26265258789063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,256,2,0,0.6953375816345215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,256,8,0,0.19710240364074708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,256,4,0,0.40738558769226074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,32,256,1,0,1.3125136375427247
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,256,16,0,0.11899360418319702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,256,32,0,0.08614559769630432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,256,64,0,0.061320000886917116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,256,4,0,0.4000703811645508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,256,8,0,0.22109599113464357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,256,2,0,0.7420767784118653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,256,16,0,0.1435472011566162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,256,32,0,0.10780960321426392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,32,256,1,0,1.3362367630004883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,256,64,0,0.08439520001411438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,512,4,0,0.9336112022399903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,512,8,0,0.4849232196807861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,512,2,0,1.7470207214355469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,512,16,0,0.321616005897522
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,512,32,0,0.19958560466766356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,512,64,0,0.1373103976249695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,32,512,1,0,3.455292892456055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,512,4,0,0.9868864059448242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,512,8,0,0.5708047866821289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,512,16,0,0.36831040382385255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,512,2,0,1.8105215072631835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,512,64,0,0.18129279613494872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,512,32,0,0.2516239881515503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,32,512,1,0,3.4539134979248045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,1024,8,0,1.3378671646118163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,1024,4,0,2.6619184494018553
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,1024,32,0,0.4882512092590332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,1024,16,0,0.7215792179107666
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,1024,64,0,0.30847361087799074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,1024,2,0,5.3174480438232425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,16,4096,2,0,31.633804321289062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,1024,4,0,2.697305679321289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,1024,8,0,1.5285008430480957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,1024,16,0,0.7905248165130615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,1024,2,0,5.218753433227539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,1024,32,0,0.5923600196838379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,32,1024,1,0,10.399513244628906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,1024,64,0,0.3954751968383789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,1536,8,0,2.6266399383544923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,32,1024,1,0,10.332273864746094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,1536,16,0,1.4330512046813966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,1536,64,0,0.5320591926574707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,1536,32,0,0.8554528236389161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,1536,4,0,5.347134399414062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,1536,2,0,11.135406494140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,1536,4,0,5.265800094604492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,16,4096,4,0,46.927789306640626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,1536,16,0,1.4607215881347657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,16,4096,1,0,65.63794555664063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,1536,64,0,0.631169605255127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,1536,8,0,2.733516883850098
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,1536,32,0,1.8398239135742187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,1536,2,0,10.852953338623047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,32,2048,16,0,2.4090015411376955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,32,2048,32,0,1.357907199859619
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,32,1536,1,0,21.90821075439453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,32,2048,64,0,0.8456416130065918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,32,2048,4,0,9.218698883056641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,32,1536,1,0,20.91456756591797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,32,2048,8,0,11.946619415283203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,16,4096,1,0,61.93515625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,32,2048,32,0,1.3686160087585448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,32,2048,64,0,0.9223360061645508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,64,1,1,0,0.016910399496555328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,1,2,0,0.015547199547290802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,1,4,0,0.013851200044155122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,1,8,0,0.013775999844074249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,1,16,0,0.01340479999780655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,1,32,0,0.0135343998670578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,1,64,0,0.013446399569511413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,32,2048,8,0,4.502286529541015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,64,1,1,0,0.022918400168418885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,1,2,0,0.02237440049648285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,1,4,0,0.019844800233840942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,1,8,0,0.02134079933166504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,1,16,0,0.019236800074577332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,1,32,0,0.020235200226306916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,1,64,0,0.019073599576950075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,16,2,0,0.08824959993362427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,32,2048,16,0,5.779001617431641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,64,16,1,0,0.15814239978790284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,32,2048,4,0,8.991223907470703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,16,4,0,0.052609598636627196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,32,2048,2,0,18.943841552734376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,16,8,0,0.07344319820404052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,16,16,0,0.02025599926710129
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,16,32,0,0.02096160054206848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,16,64,0,0.017638400197029114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,16,8,0,0.04300160109996796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,16,4,0,0.06652479767799377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,16,2,0,0.09870399832725525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,64,16,1,0,0.3376143932342529
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,16,32,0,0.0277072012424469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,16,16,0,0.028007999062538147
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,16,64,0,0.02460159957408905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,32,2,0,0.16671680212020873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,32,4,0,0.09162880182266235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,32,16,0,0.03651520013809204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,32,8,0,0.05788319706916809
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,64,32,1,0,0.4780479907989502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,32,32,0,0.023259200155735016
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,32,64,0,0.023244799673557283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,32,4,0,0.10849440097808838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,32,8,0,0.09890720248222351
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,32,2,0,0.16752159595489502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,64,32,1,0,0.3015183925628662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,32,16,0,0.05018879771232605
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,32,32,0,0.05107839703559876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,32,64,0,0.03208959996700287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,64,8,0,0.09883840084075927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,64,2,0,0.3072927951812744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,64,4,0,0.20806241035461426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,64,16,0,0.06255840063095093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,64,32,0,0.0429504007101059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,64,64,0,0.03847039937973022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,64,64,1,0,0.5589263916015625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,64,4,0,0.23133440017700196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,64,8,0,0.11221920251846314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,64,2,0,0.30686559677124026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,64,32,0,0.05814560055732727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,64,16,0,0.09523680210113525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,64,64,1,0,0.5826111793518066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,64,64,0,0.0438944011926651
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,128,8,0,0.1793503999710083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,128,4,0,0.3187792062759399
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,128,16,0,0.11209280490875244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,128,64,0,0.05623679757118225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,128,32,0,0.07681120038032532
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,128,2,0,0.760145616531372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,64,128,1,0,1.1498384475708008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,128,2,0,0.631550407409668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,128,4,0,0.47255840301513674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,128,8,0,0.2040544033050537
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,128,16,0,0.13687679767608643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,128,32,0,0.1406272053718567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,128,64,0,0.08062880039215088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,64,128,1,0,1.1761712074279784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,256,8,0,0.3806096076965332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,256,4,0,0.7417679786682129
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,256,16,0,0.22477920055389405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,256,32,0,0.14836159944534302
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,256,64,0,0.11052160263061524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,32,2048,2,0,17.76923828125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,256,2,0,2.304476737976074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,64,256,1,0,2.6093231201171876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,256,4,0,0.8165040016174316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,256,2,0,1.4376704216003418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,256,16,0,0.2712656021118164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,256,64,0,0.15565919876098633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,256,32,0,0.19743039608001708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,256,8,0,0.7928175926208496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,64,256,1,0,2.677024078369141
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,512,8,0,0.9589296340942383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,512,64,0,0.25414879322052003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,512,16,0,1.0546799659729005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,512,32,0,0.38762879371643066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,512,4,0,1.8918144226074218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,512,8,0,1.0367520332336426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,512,2,0,3.5559425354003906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,512,4,0,1.9750688552856446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,512,16,0,0.7048399925231934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,512,32,0,0.4703055858612061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,512,64,0,0.3391200065612793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,64,512,1,0,6.938692474365235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,64,512,1,0,6.865345764160156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,32,2048,1,0,36.67341613769531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,512,2,0,7.389619445800781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,64,1024,8,0,2.751304054260254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,64,1024,4,0,5.510030364990234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,64,1024,32,0,0.9666784286499024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,64,1024,64,0,0.9061200141906738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,64,1024,16,0,1.438584041595459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,64,1024,16,0,1.5767215728759765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,64,1024,32,0,1.0872639656066894
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,32,2048,1,0,34.959225463867185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,64,1024,2,0,10.65557632446289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,64,1024,64,0,0.7391520023345948
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,1,4,0,0.01579679995775223
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,128,1,1,0,0.03142400085926056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,1,2,0,0.018211199343204497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,1,8,0,0.016019199788570405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,1,16,0,0.016574400663375854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,1,32,0,0.015566399693489075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,1,64,0,0.015398399531841278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,64,1024,4,0,5.626744079589844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,128,1,1,0,0.035950401425361635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,64,1024,8,0,5.93939208984375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,1,4,0,0.022383999824523926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,1,2,0,0.02497279942035675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,1,8,0,0.050040000677108766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,1,32,0,0.02220800071954727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,1,16,0,0.021673600375652313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,1,64,0,0.050083202123641965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,16,4,0,0.14244799613952636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,128,16,1,0,0.2937808036804199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,16,2,0,0.15910240411758422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,16,8,0,0.08220639824867249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,16,16,0,0.035601601004600525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,16,64,0,0.03436799943447113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,16,32,0,0.023708799481391908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,16,2,0,0.21783359050750734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,128,16,1,0,0.3008671998977661
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,16,4,0,0.10498080253601075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,16,8,0,0.08324159979820252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,16,16,0,0.04941760003566742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,16,32,0,0.03488959968090057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,16,64,0,0.04082399904727936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,32,4,0,0.18064479827880858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,32,2,0,0.2971679925918579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,32,8,0,0.10982240438461303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,128,32,1,0,0.5925583839416504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,32,32,0,0.0463808000087738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,32,16,0,0.06302559971809388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,32,64,0,0.03014880120754242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,32,4,0,0.1783519983291626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,32,2,0,0.314086389541626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,32,8,0,0.11415519714355468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,128,32,1,0,0.6378767967224122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,32,32,0,0.059115201234817505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,32,16,0,0.0796176016330719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,32,64,0,0.059627199172973634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,64,1024,2,0,10.428803253173829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,64,4,0,0.39297759532928467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,64,2,0,0.579582405090332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,64,32,0,0.09427679777145385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,64,16,0,0.11096960306167603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,64,8,0,0.18068000078201293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,64,64,0,0.05519199967384338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,128,64,1,0,1.1434927940368653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,64,16,0,0.13484159708023072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,64,4,0,0.33539040088653566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,64,2,0,0.600984001159668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,64,8,0,0.2090480089187622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,64,32,0,0.09966239929199219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,64,64,0,0.07988479733467102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,128,64,1,0,1.4814319610595703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,128,4,0,0.6167056083679199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,128,8,0,0.3483680009841919
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,128,32,0,0.14086079597473145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,128,64,0,0.10607359409332276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,128,16,0,0.330182409286499
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,128,2,0,1.1957615852355956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,128,8,0,0.38703839778900145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,128,16,0,0.25096960067749025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,128,128,1,0,2.2872047424316406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,128,32,0,0.1847599983215332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,128,4,0,0.9742527961730957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,128,64,0,0.14724639654159546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,128,2,0,1.2534079551696777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,64,1024,1,0,21.781265258789062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,256,16,0,0.4335023880004883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,256,8,0,0.7451424121856689
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,128,128,1,0,2.346552085876465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,256,64,0,0.20288000106811524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,256,32,0,0.28695681095123293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,256,4,0,2.5648239135742186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,256,8,0,0.8355312347412109
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,256,2,0,2.759440040588379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,256,4,0,1.5510160446166992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,256,32,0,0.3624943971633911
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,256,64,0,0.28664159774780273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,256,16,0,0.8887215614318847
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,256,2,0,2.8316928863525392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,128,256,1,0,5.216392135620117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,128,512,16,0,1.2299311637878418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,64,1024,1,0,21.405804443359376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,128,512,32,0,0.7335199832916259
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,128,256,1,0,5.348713684082031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,128,512,64,0,0.48511037826538084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,128,512,4,0,3.7035377502441404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,128,512,8,0,4.010712051391602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,128,512,8,0,2.071940803527832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,128,512,16,0,1.3803888320922852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,256,1,1,0,0.05429279804229736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,1,2,0,0.027348798513412476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,128,512,32,0,1.5500975608825684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,1,4,0,0.022350400686264038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,1,16,0,0.02195200026035309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,128,512,64,0,0.6432928085327149
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,1,8,0,0.038217601180076596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,1,32,0,0.02194560021162033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,1,2,0,0.03383679986000061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,1,64,0,0.02271360009908676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,256,1,1,0,0.08341919779777526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,1,4,0,0.028569599986076354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,128,512,2,0,7.214244842529297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,1,8,0,0.028832000494003297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,128,512,4,0,3.8558624267578123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,1,16,0,0.02791520059108734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,1,64,0,0.028540799021720888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,1,32,0,0.030324798822402955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,16,2,0,0.29257280826568605
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,16,4,0,0.16600799560546875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,16,8,0,0.10293760299682617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,16,16,0,0.06405280232429504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,16,32,0,0.04257920086383819
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,256,16,1,0,0.5685823917388916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,16,64,0,0.03566559851169586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,16,2,0,0.3122607946395874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,16,4,0,0.18064800500869752
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,16,8,0,0.11424479484558106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,256,16,1,0,0.580072021484375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,16,16,0,0.08070880174636841
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,16,32,0,0.05988479852676391
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,16,64,0,0.050414401292800906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,32,8,0,0.17963199615478515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,32,4,0,0.3081568002700806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,32,2,0,0.5891119956970214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,32,32,0,0.07650240063667298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,32,16,0,0.1129871964454651
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,32,64,0,0.05541120171546936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,128,512,2,0,7.154476928710937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,256,32,1,0,1.132431983947754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,32,4,0,0.33579680919647215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,32,2,0,0.6120416164398194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,32,32,0,0.10044480562210083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,32,16,0,0.13559199571609498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,32,8,0,0.2037584066390991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,32,64,0,0.08112319707870483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,256,32,1,0,1.131486415863037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,64,8,0,0.33974080085754393
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,64,16,0,0.208951997756958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,64,4,0,0.6093855857849121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,64,32,0,0.1403887987136841
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,64,64,0,0.10499199628829955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,64,2,0,1.1386544227600097
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,64,8,0,0.3793600082397461
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,64,4,0,0.6424911975860595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,64,16,0,0.249236798286438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,64,32,0,0.18428159952163697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,256,64,1,0,2.224407958984375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,64,2,0,1.1816783905029298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,128,512,1,0,13.90718994140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,64,64,0,0.14737279415130616
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,128,16,0,0.3985840082168579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,256,64,1,0,2.2622032165527344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,128,8,0,0.6813295841217041
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,128,32,0,0.26298720836639405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,128,4,0,1.2138912200927734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,128,64,0,0.2006848096847534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,128,8,0,0.7463183879852295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,128,2,0,2.3713327407836915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,128,16,0,0.5044400215148925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,128,4,0,1.2919872283935547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,128,32,0,0.34219679832458494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,128,64,0,0.2761791944503784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,128,512,1,0,14.329409790039062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,128,2,0,2.4389776229858398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,256,128,1,0,4.5429023742675785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,4,256,256,32,0,0.5363552093505859
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,8,256,256,16,0,0.8585984230041503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,16,256,256,8,0,1.4799519538879395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,2,256,256,64,0,0.3899280071258545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,32,256,256,4,0,2.9208335876464844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,256,128,1,0,4.668203353881836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,16,256,256,8,0,1.640073585510254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,4,256,256,32,0,0.6971360206604004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,8,256,256,16,0,0.9983679771423339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,2,256,256,64,0,0.5595424175262451
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,32,256,256,4,0,3.0990016937255858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,64,256,256,2,0,5.440006256103516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,64,256,256,2,0,5.63087043762207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,fp8,128,256,256,1,0,10.519416046142577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,mla_context,flash_attention,float16,float16,128,256,256,1,0,29.615521240234376
