framework,version,device,op_name,kernel_source,mla_dtype,kv_cache_dtype,num_heads,batch_size,isl,tp_size,step,latency
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,1,1,0,0.012966400384902954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,1,2,0,0.018287999927997588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,1,4,0,0.017953599989414214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,1,8,0,0.017793600261211396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,1,16,0,0.01797440052032471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,1,32,0,0.01796800047159195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,1,64,0,0.01791359931230545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,16,1,0,0.014003199338912965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,16,2,0,0.014214399456977844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,16,4,0,0.013831999897956849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,16,8,0,0.013801600039005279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,16,16,0,0.013527999818325042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,16,32,0,0.013031999766826629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,16,64,0,0.012782399356365205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,1,8,0,0.012145599722862244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,1,64,0,0.012161599844694138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,16,1,0,0.02003840059041977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,16,8,0,0.01969760060310364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,1,16,0,0.012303999811410903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,1,1,0,0.018857599794864656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,16,64,0,0.018751999735832213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,32,1,0,0.015305599570274353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,32,2,0,0.01404159963130951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,32,4,0,0.014059199392795563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,32,8,0,0.014047999680042268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,32,16,0,0.014030399918556213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,32,32,0,0.013655999302864074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,32,64,0,0.01313599944114685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,32,1,0,0.021499200165271758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,32,2,0,0.02014880031347275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,32,4,0,0.019964799284934998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,1,4,0,0.012204799801111221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,32,8,0,0.020024000108242034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,32,32,0,0.019571200013160706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,32,64,0,0.019011199474334717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,32,16,0,0.01951040029525757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,64,1,0,0.015460799634456634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,64,2,0,0.015086400508880615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,64,4,0,0.014052799344062806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,64,8,0,0.013872000575065612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,64,16,0,0.013889600336551667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,64,32,0,0.013716800510883332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,64,64,0,0.013711999356746673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,64,1,0,0.02205280065536499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,64,2,0,0.021611200273036958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,64,4,0,0.020003199577331543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,64,8,0,0.020287999510765077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,64,32,0,0.019920000433921815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,64,64,0,0.019571200013160706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,128,1,0,0.01804320067167282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,128,2,0,0.01727360039949417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,128,4,0,0.016908800601959227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,16,16,0,0.0192208006978035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,128,16,0,0.015695999562740325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,128,32,0,0.015556800365447997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,128,64,0,0.014643199741840363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,128,1,0,0.024604800343513488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,128,2,0,0.023454399406909944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,128,4,0,0.023292799293994904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,128,8,0,0.021775999665260316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,128,16,0,0.021835200488567352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,128,32,0,0.021529600024223328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,128,64,0,0.020638400316238405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,256,1,0,0.030803200602531434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,1,2,0,0.01263359934091568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,256,2,0,0.02056639939546585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,256,8,0,0.019310399889945984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,256,4,0,0.02001280039548874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,256,32,0,0.018990400433540344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,256,64,0,0.017822399735450745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,256,1,0,0.04134880006313324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,256,2,0,0.027475199103355406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,64,16,0,0.019964799284934998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,256,4,0,0.026679998636245726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,256,8,0,0.02499839961528778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,256,16,0,0.026267200708389282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,256,32,0,0.024710400402545928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,256,64,0,0.023713600635528565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,512,1,0,0.07015039920806884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,512,2,0,0.03843039870262146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,512,4,0,0.02709600031375885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,1,32,0,0.012321600317955017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,16,4,0,0.019731199741363524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,128,8,0,0.01592639982700348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,512,16,0,0.028177601099014283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,512,1,0,0.08119360208511353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,512,64,0,0.022622400522232057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,512,4,0,0.03463039994239807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,512,8,0,0.045100799202919005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,512,32,0,0.03094559907913208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,512,16,0,0.03601120114326477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,1024,1,0,0.18112319707870483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,1024,2,0,0.0969983994960785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,1024,4,0,0.05201119780540466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,1024,8,0,0.04129599928855896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,1024,16,0,0.044147199392318724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,256,16,0,0.01897439956665039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,1024,32,0,0.032016000151634215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,1024,64,0,0.027535998821258546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,1024,2,0,0.10467040538787842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,16,2,0,0.019684800505638124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,16,32,0,0.018673600256443025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,1024,4,0,0.06153280138969421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,512,32,0,0.023856000602245332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,512,2,0,0.047651201486587524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,1024,8,0,0.04832319915294647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,1024,64,0,0.03482399880886078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,1536,2,0,0.17448799610137938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,1536,4,0,0.09653120040893555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,1536,8,0,0.055155199766159055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,1536,16,0,0.06099039912223816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,1536,32,0,0.0434143990278244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,1536,64,0,0.03426559865474701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,1536,1,0,0.34236481189727785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,1024,1,0,0.1886847972869873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,1536,2,0,0.184334397315979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,1536,4,0,0.1073904037475586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,1536,8,0,0.06470080018043518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,1536,16,0,0.07280799746513367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,1536,32,0,0.05472639799118042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,1536,64,0,0.04441919922828674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,2048,2,0,0.27975680828094485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,2048,1,0,0.5330527782440185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,2048,4,0,0.144924795627594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,2048,8,0,0.07853919863700867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,512,8,0,0.03564000129699707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,2048,16,0,0.06834719777107238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,2048,32,0,0.05944960117340088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,2048,64,0,0.041552001237869264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,2048,4,0,0.1571328043937683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,2048,2,0,0.2874959945678711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,2048,1,0,0.5446383953094482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,2048,16,0,0.07805119752883911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,512,64,0,0.02974559962749481
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,2048,64,0,0.05418720245361328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,3072,2,0,0.5539040088653564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,3072,1,0,1.0642560005187989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,3072,4,0,0.28257920742034914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,3072,16,0,0.0970687985420227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,3072,8,0,0.16028319597244262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,3072,32,0,0.08154720067977905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,3072,64,0,0.061105602979660036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,3072,2,0,0.5562160015106201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,3072,4,0,0.2992991924285889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,3072,8,0,0.1783087968826294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,3072,16,0,0.11124320030212402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,3072,32,0,0.09748160243034362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,2048,8,0,0.09145119786262512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,1536,1,0,0.3323647975921631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,2048,32,0,0.07211199998855591
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,4096,8,0,0.25114400386810304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,4096,4,0,0.47423357963562013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,4096,16,0,0.13674240112304686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,4096,32,0,0.12315200567245484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,4096,2,0,0.9397359848022461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,4096,64,0,0.09155200123786926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,4096,4,0,0.4881743907928467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,4096,2,0,0.9326016426086425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,4096,16,0,0.15034559965133668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,4096,32,0,0.13882240056991577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,4096,1,0,1.7797296524047852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,3072,1,0,1.0704640388488769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,4096,64,0,0.1080399990081787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,1024,16,0,0.053198397159576416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,3072,64,0,0.07603359818458558
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,6144,16,0,0.29099359512329104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,6144,4,0,1.008187198638916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,6144,32,0,0.18825279474258422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,6144,64,0,0.13121600151062013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,6144,2,0,2.035108757019043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,4096,1,0,1.7910848617553712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,4096,8,0,0.26489920616149903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,6144,4,0,1.080454444885254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,6144,8,0,0.5153391838073731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,6144,16,0,0.3120208024978638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,6144,2,0,2.0040800094604494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,6144,64,0,0.1508895993232727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,6144,1,0,4.235158538818359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,1024,32,0,0.04012799859046936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,6144,8,0,0.5091487884521484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,8192,16,0,0.4498000144958496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,8192,32,0,0.24390718936920167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,8192,8,0,0.9553055763244629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,6144,1,0,3.959462356567383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,8192,64,0,0.2351855993270874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,8192,4,0,1.74029598236084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,8192,16,0,0.5159840106964111
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,8192,8,0,0.8680352210998535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,8192,32,0,0.2892688035964966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,8192,64,0,0.25577759742736816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,8192,4,0,1.8518720626831056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,8192,2,0,3.7536815643310546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,8192,2,0,3.5245521545410154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,10240,8,0,1.4263936042785645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,10240,32,0,0.4322207927703857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,10240,16,0,0.7461999893188477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,10240,64,0,0.3154031991958618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,10240,4,0,2.81823673248291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,8192,1,0,7.4979393005371096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,6144,32,0,0.21982879638671876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,8192,1,0,7.184404754638672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,10240,2,0,5.747947311401367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,10240,8,0,1.4584400177001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,10240,32,0,0.4660783767700195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,10240,4,0,2.8864688873291016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,10240,2,0,5.6315361022949215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,10240,64,0,0.3388688087463379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,10240,16,0,0.7385791778564453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,10240,1,0,11.283316802978515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,12288,16,0,0.9676848411560058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,12288,32,0,0.6053247928619385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,12288,64,0,0.34979519844055174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,12288,8,0,1.9261648178100585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,12288,4,0,4.120334243774414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,10240,1,0,10.868851470947266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,12288,16,0,1.068289566040039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,12288,2,0,8.274244689941407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,12288,64,0,0.40140318870544434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,12288,4,0,3.855846405029297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,12288,8,0,2.08843994140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,12288,32,0,0.6119679927825927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,12288,2,0,7.955486297607422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,16384,16,0,1.6925504684448243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,16384,32,0,0.8912768363952637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,16384,64,0,0.4887392044067383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,16384,8,0,3.4761566162109374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,12288,1,0,16.32948455810547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,16384,4,0,7.109760284423828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,16384,16,0,1.8138368606567383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,16384,32,0,0.9442704200744629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,16384,8,0,3.4516063690185548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,16384,64,0,0.5010992050170898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,12288,1,0,15.720793151855469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,16384,4,0,7.050105285644531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,16384,2,0,15.637818908691406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,16384,2,0,13.915423583984374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,32768,16,0,7.345887756347656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,32768,32,0,3.611918258666992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,1,32768,64,0,1.9044815063476563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,32768,8,0,14.621528625488281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,16384,1,0,31.323919677734374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,16384,1,0,28.895794677734376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,32768,16,0,7.167062377929687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,32768,4,0,28.81131591796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,1,32768,64,0,1.8524320602416993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,32768,8,0,13.652976989746094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,1,1,0,0.013463999330997466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,1,4,0,0.012825599312782288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,1,2,0,0.013612799346446991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,1,16,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,1,32,0,0.012695999443531036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,1,64,0,0.012771199643611907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,1,1,0,0.01913439929485321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,1,2,0,0.018636800348758698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,1,4,0,0.01767839938402176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,32768,32,0,3.492243194580078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,1,8,0,0.017451199889183044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,1,32,0,0.017566399276256563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,1,16,0,0.018675200641155243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,1,64,0,0.017665599286556245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,16,1,0,0.016051200032234193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,16,2,0,0.015166400372982025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,16,4,0,0.014368000626564025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,16,8,0,0.013913600146770478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,16,16,0,0.01356000006198883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,16,32,0,0.01292639970779419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,16,64,0,0.012878400087356568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,16,1,0,0.021209600567817687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,16,2,0,0.02005600035190582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,16,4,0,0.01931840032339096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,16,8,0,0.019844800233840942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,16,16,0,0.019147199392318726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,16,32,0,0.018828800320625304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,16,64,0,0.018328000605106354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,32,1,0,0.015119999647140503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,32,2,0,0.015187199413776397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,32,4,0,0.01412159949541092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,32,8,0,0.01377120018005371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,32,16,0,0.013896000385284425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,32,32,0,0.013289600610733032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,32,64,0,0.012807999551296235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,32,1,0,0.021451200544834136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,32,2,0,0.021006399393081666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,32,4,0,0.01961600035429001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,32,8,0,0.019631999731063842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,32,16,0,0.019625599682331085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,32,32,0,0.019036799669265747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,32,64,0,0.018614399433135986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,64,1,0,0.016462400555610657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,64,2,0,0.015904000401496886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,64,4,0,0.015430399775505066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,64,8,0,0.015124799311161041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,64,16,0,0.013782399892807006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,64,32,0,0.01395999938249588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,64,64,0,0.013388800621032714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,1,8,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,64,2,0,0.02163040041923523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,64,4,0,0.021430400013923646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,64,16,0,0.019814400374889372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,64,32,0,0.019633600115776063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,64,1,0,0.02293439954519272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,64,64,0,0.01974399983882904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,128,1,0,0.03002080023288727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,128,2,0,0.018080000579357148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,128,4,0,0.01743679940700531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,128,8,0,0.016920000314712524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,128,16,0,0.01676799952983856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,128,32,0,0.01583999991416931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,128,64,0,0.01499519944190979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,128,1,0,0.036052799224853514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,128,2,0,0.024611200392246246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,128,4,0,0.023545600473880768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,128,8,0,0.022889600694179536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,128,16,0,0.022862400114536285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,128,32,0,0.021908800303936004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,128,64,0,0.02088160067796707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,256,1,0,0.057659202814102174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,256,2,0,0.03195840120315552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,256,4,0,0.02117439955472946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,256,8,0,0.020747199654579163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,256,16,0,0.020608000457286835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,256,32,0,0.01980320066213608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,256,64,0,0.019073599576950075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,256,1,0,0.06808320283889771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,256,2,0,0.04058560132980347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,64,8,0,0.01996160000562668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,256,4,0,0.028191998600959778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,256,8,0,0.027505600452423097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,256,16,0,0.027132800221443175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,256,32,0,0.026204800605773924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,256,64,0,0.025143998861312866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,512,2,0,0.07046080231666565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,512,1,0,0.12705279588699342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,512,4,0,0.03982560038566589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,512,8,0,0.033934399485588074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,512,16,0,0.037985599040985106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,512,32,0,0.030131199955940248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,512,64,0,0.02566399872303009
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,512,1,0,0.13784799575805665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,512,8,0,0.04158560037612915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,512,16,0,0.04843200147151947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,512,32,0,0.03845599889755249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,512,64,0,0.03296000063419342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,1024,1,0,0.3380800008773804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,1024,2,0,0.17807040214538575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,1024,4,0,0.09566559791564941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,1024,8,0,0.05445280075073242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,1024,16,0,0.049423998594284056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,1024,32,0,0.04714399874210358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,1024,64,0,0.035873600840568544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,1024,1,0,0.3495568037033081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,1024,2,0,0.18895360231399536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,1024,4,0,0.10729759931564331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,32768,4,0,29.34477844238281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,1024,8,0,0.06746240258216858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,512,2,0,0.08097919821739197
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,1024,32,0,0.05963839888572693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,1024,64,0,0.04796159863471985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,1536,2,0,0.3336800098419189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,1536,1,0,0.6469823837280273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,1536,4,0,0.1740880012512207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,1536,8,0,0.0989520013332367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,1536,16,0,0.0667087972164154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,1536,32,0,0.07387679815292358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,1536,64,0,0.0488783985376358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,1536,2,0,0.3443072080612183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,1536,1,0,0.6596464157104492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,1536,4,0,0.18689119815826416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,1536,8,0,0.11307519674301147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,1536,32,0,0.08834559917449951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,1536,64,0,0.06403200030326843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,512,4,0,0.05027520060539246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,1024,16,0,0.060438400506973265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,2048,1,0,1.0539679527282715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,2048,8,0,0.15747679471969606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,2048,4,0,0.2806031942367554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,2048,2,0,0.5376160144805908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,2048,16,0,0.0887935996055603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,2048,32,0,0.07948319911956787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,2048,64,0,0.06663200259208679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,2048,4,0,0.29109599590301516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,2048,2,0,0.5522223949432373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,2048,8,0,0.16548160314559937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,2048,16,0,0.10182240009307861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,2048,32,0,0.09466239809989929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,2048,64,0,0.08392000198364258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,1536,16,0,0.0809503972530365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,3072,4,0,0.5549215793609619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,3072,2,0,1.119331169128418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,3072,8,0,0.29144160747528075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,3072,16,0,0.18171199560165405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,3072,32,0,0.11610239744186401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,3072,1,0,2.113559913635254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,32768,2,0,59.3880615234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,3072,2,0,1.1840720176696777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,2048,1,0,1.0871888160705567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,3072,4,0,0.659555196762085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,3072,8,0,0.32458720207214353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,3072,1,0,2.1806512832641602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,3072,32,0,0.15443840026855468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,3072,64,0,0.11080319881439209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,4096,2,0,1.9021472930908203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,3072,16,0,0.19122719764709473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,3072,64,0,0.12787519693374633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,4096,1,0,3.9876449584960936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,4096,16,0,0.26073920726776123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,4096,8,0,0.48456320762634275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,4096,32,0,0.16267039775848388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,4096,64,0,0.14510079622268676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,4096,4,0,0.9925583839416504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,4096,8,0,0.5397344112396241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,4096,4,0,0.9520095825195313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,4096,16,0,0.30112481117248535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,4096,32,0,0.17257280349731446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,4096,64,0,0.17826399803161622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,4096,2,0,1.9396064758300782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,4096,1,0,3.758524703979492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,6144,8,0,1.016164779663086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,6144,4,0,2.1294479370117188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,32768,2,0,56.60107421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,6144,32,0,0.3210495948791504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,6144,2,0,4.349067306518554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,6144,16,0,0.5638720035552979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,6144,64,0,0.24101440906524657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,6144,2,0,4.117911911010742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,6144,16,0,0.5625552177429199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,6144,8,0,1.0237872123718261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,6144,32,0,0.36983680725097656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,6144,64,0,0.2468480110168457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,6144,1,0,8.557057952880859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,6144,4,0,2.144231986999512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,8192,16,0,0.9126336097717285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,8192,4,0,3.5886913299560548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,6144,1,0,8.726396942138672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,8192,32,0,0.487608003616333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,8192,64,0,0.29910879135131835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,8192,8,0,1.80882568359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,8192,2,0,7.397822570800781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,8192,4,0,3.6145824432373046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,8192,8,0,1.718307113647461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,8192,64,0,0.3179136037826538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,8192,16,0,1.0010671615600586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,8192,32,0,0.5549183845520019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,8192,2,0,7.3612525939941404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,8192,1,0,14.978196716308593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,10240,8,0,2.8635839462280273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,10240,4,0,5.936452865600586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,10240,64,0,0.45578718185424805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,10240,32,0,0.8611151695251464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,10240,16,0,1.4526975631713868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,8192,1,0,14.775328063964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,10240,2,0,11.44240493774414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,10240,8,0,2.789654350280762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,10240,4,0,5.489702224731445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,10240,64,0,0.48676481246948244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,10240,32,0,0.8379280090332031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,10240,16,0,1.4945311546325684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,10240,2,0,11.260858917236328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,10240,1,0,22.746922302246094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,12288,8,0,4.070158386230469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,12288,32,0,1.106174373626709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,12288,64,0,0.6272192001342773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,12288,16,0,2.0272783279418944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,12288,4,0,8.58298568725586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,10240,1,0,23.489576721191405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,12288,2,0,17.423800659179687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,32768,1,0,116.91510009765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,12288,4,0,7.986840057373047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,12288,16,0,2.1782880783081056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,12288,8,0,4.119918441772461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,12288,64,0,0.6814223766326905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,12288,32,0,1.1673808097839355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,32768,1,0,111.68038330078124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,12288,2,0,17.16441650390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,16384,8,0,7.861402893066407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,16384,64,0,1.0229248046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,16384,32,0,1.7913503646850586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,16384,16,0,3.5515182495117186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,12288,1,0,34.271340942382814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,16384,4,0,14.878945922851562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,16384,16,0,4.062952041625977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,16384,8,0,7.00799331665039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,16384,32,0,1.9077775955200196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,16384,64,0,1.0172911643981934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,12288,1,0,32.525167846679686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,16384,4,0,14.512745666503907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,16384,2,0,30.113446044921876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,16384,2,0,28.3807373046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,32768,16,0,14.950851440429688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,32768,32,0,7.587206268310547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,2,32768,64,0,3.7366527557373046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,32768,8,0,29.396591186523438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,16384,1,0,60.30665893554688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,16384,1,0,57.2619873046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,32768,16,0,14.741828918457031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,32768,4,0,59.807275390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,2,32768,64,0,3.6980224609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,1,1,0,0.01435839980840683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,1,2,0,0.013801600039005279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,1,4,0,0.013663999736309052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,32768,32,0,7.281307220458984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,1,8,0,0.013630400598049163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,1,16,0,0.013680000603199006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,1,32,0,0.013595199584960938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,1,64,0,0.01358560025691986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,1,1,0,0.020572799444198608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,1,2,0,0.019952000677585603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,1,4,0,0.018603199720382692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,1,8,0,0.018507200479507446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,1,16,0,0.018537600338459016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,1,32,0,0.018691200017929076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,1,64,0,0.018662400543689728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,16,1,0,0.015803200006484986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,16,2,0,0.015617600083351136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,16,4,0,0.014486399292945863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,16,8,0,0.014348800480365752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,16,16,0,0.013872000575065612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,16,32,0,0.013283200562000275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,16,64,0,0.013046400249004364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,16,1,0,0.022470399737358093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,16,2,0,0.021447999775409697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,16,4,0,0.020103999972343446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,16,8,0,0.01992799937725067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,16,16,0,0.019480000436306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,16,32,0,0.018934400379657747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,16,64,0,0.01857440024614334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,32768,8,0,28.7858642578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,32,2,0,0.01563359946012497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,32,4,0,0.015502400696277618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,32,8,0,0.014244799315929414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,32,16,0,0.014049600064754485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,32,32,0,0.013577599823474885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,32,64,0,0.013238400220870972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,32,1,0,0.023217600584030152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,32,2,0,0.021963199973106383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,32,4,0,0.021827200055122377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,32,8,0,0.02030719965696335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,32,16,0,0.020164799690246583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,32,32,0,0.0197952002286911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,32,64,0,0.01921280026435852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,64,1,0,0.02815040051937103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,64,2,0,0.0164000004529953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,64,4,0,0.015787200629711153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,64,8,0,0.015740799903869628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,64,16,0,0.01443679928779602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,64,32,0,0.01446560025215149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,64,64,0,0.0141184002161026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,64,1,0,0.03610239923000336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,64,2,0,0.0232464000582695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,32,1,0,0.01616320013999939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,64,4,0,0.022460800409317017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,64,8,0,0.022158400714397432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,64,16,0,0.020768000185489653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,64,32,0,0.020688000321388244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,64,64,0,0.02030559927225113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,128,1,0,0.05194079875946045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,128,4,0,0.01844480037689209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,128,8,0,0.01785759925842285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,128,16,0,0.01757279932498932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,128,32,0,0.016935999691486358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,128,64,0,0.01610559970140457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,128,1,0,0.06194080114364624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,128,2,0,0.039868798851966855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,128,4,0,0.02515679895877838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,128,8,0,0.024560000002384185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,128,16,0,0.02412479966878891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,128,32,0,0.023391999304294586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,128,64,0,0.022152000665664674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,256,1,0,0.1021888017654419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,256,2,0,0.05910400152206421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,256,4,0,0.0351936012506485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,256,8,0,0.022729599475860597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,256,16,0,0.02242079973220825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,256,32,0,0.022337600588798523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,256,64,0,0.02126079946756363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,256,1,0,0.11128799915313721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,256,2,0,0.07045279741287232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,256,4,0,0.04668799936771393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,256,8,0,0.030368000268936157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,256,16,0,0.02984800040721893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,256,32,0,0.029728001356124877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,256,64,0,0.027476799488067628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,512,2,0,0.1276047945022583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,512,1,0,0.23734560012817382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,512,4,0,0.07211520075798035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,512,8,0,0.055764800310134886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,512,16,0,0.036616000533103946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,512,32,0,0.04092000126838684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,512,64,0,0.033078399300575254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,512,2,0,0.14057600498199463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,512,4,0,0.08462560176849365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,512,8,0,0.07002720236778259
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,512,16,0,0.04742879867553711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,512,32,0,0.053814399242401126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,512,64,0,0.04364640116691589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,1024,1,0,0.658569622039795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,1024,2,0,0.34576480388641356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,128,2,0,0.029423999786376952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,1024,8,0,0.10199840068817138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,1024,16,0,0.08066080212593078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,512,1,0,0.2495568037033081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,1024,32,0,0.05550240278244019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,1024,64,0,0.05449119806289673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,1024,1,0,0.672492790222168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,1024,2,0,0.36034719944000243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,1024,4,0,0.1978816032409668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,1024,8,0,0.11872320175170899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,1024,4,0,0.18148159980773926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,1024,16,0,0.0967519998550415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,1024,32,0,0.07114400267601013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,1024,64,0,0.07164160013198853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,1536,2,0,0.6657760143280029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,1536,4,0,0.34158399105072024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,1536,1,0,1.2808943748474122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,1536,8,0,0.183787202835083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,1536,16,0,0.1239359974861145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,1536,32,0,0.08956000208854675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,1536,64,0,0.08630399703979492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,32768,4,0,56.935369873046874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,1536,2,0,0.6749279975891114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,1536,8,0,0.20375521183013917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,1536,4,0,0.39875359535217286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,1536,1,0,1.2947855949401856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,1536,16,0,0.14345439672470092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,1536,64,0,0.10915039777755738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,1536,32,0,0.11640959978103638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,2048,4,0,0.5469295978546143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,2048,8,0,0.2889359951019287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,2048,2,0,1.1041872024536132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,2048,16,0,0.16262079477310182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,2048,32,0,0.1337839961051941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,2048,64,0,0.09436479806900025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,2048,1,0,2.0871824264526366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,2048,4,0,0.5966944217681884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,2048,2,0,1.0754287719726563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,2048,8,0,0.3142575979232788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,2048,16,0,0.19868479967117308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,2048,32,0,0.15169119834899902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,2048,64,0,0.12054239511489868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,2048,1,0,2.1387887954711915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,3072,8,0,0.6087279796600342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,3072,4,0,1.1124688148498536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,3072,32,0,0.21161599159240724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,3072,16,0,0.33420159816741946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,3072,64,0,0.15684319734573365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,3072,2,0,2.1934864044189455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,3072,4,0,1.2192079544067382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,3072,8,0,0.6330272197723389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,3072,2,0,2.1839584350585937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,3072,1,0,4.6455951690673825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,3072,32,0,0.24124000072479249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,3072,64,0,0.19688160419464112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,3072,1,0,4.52630386352539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,4096,4,0,1.9875728607177734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,4096,2,0,3.972512054443359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,4096,16,0,0.5522912025451661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,4096,8,0,1.0702207565307618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,4096,32,0,0.3023263931274414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,4096,64,0,0.252726411819458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,3072,16,0,0.35136640071868896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,4096,1,0,8.059937286376954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,4096,8,0,1.0759872436523437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,4096,2,0,3.9557216644287108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,4096,32,0,0.3492464065551758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,4096,16,0,0.57882080078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,4096,64,0,0.28343520164489744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,4096,1,0,7.699459075927734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,4096,4,0,1.8731119155883789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,6144,8,0,2.0398303985595705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,6144,16,0,1.124283218383789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,6144,32,0,0.6319327831268311
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,6144,4,0,4.447073745727539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,32768,2,0,116.8911865234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,6144,64,0,0.40849599838256834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,6144,2,0,8.725820922851563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,6144,8,0,2.211471939086914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,6144,4,0,4.124585723876953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,6144,16,0,1.1833824157714843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,6144,64,0,0.4626704216003418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,6144,2,0,8.604542541503907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,6144,32,0,0.6520800113677978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,6144,1,0,17.237693786621094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,8192,8,0,4.34619026184082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,8192,16,0,2.1296384811401365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,8192,4,0,7.439568328857422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,6144,1,0,17.388203430175782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,8192,64,0,0.5502016067504882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,8192,32,0,1.0838399887084962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,32768,2,0,114.11788330078124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,8192,2,0,15.331123352050781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,8192,4,0,7.36120834350586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,8192,16,0,1.9217439651489259
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,8192,64,0,0.6139232158660889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,8192,8,0,3.635852813720703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,8192,32,0,1.067416000366211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,8192,2,0,14.511595153808594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,8192,1,0,30.988571166992188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,10240,8,0,5.930388641357422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,10240,32,0,1.546177577972412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,10240,16,0,3.0504703521728516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,10240,64,0,0.9367280006408691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,10240,4,0,11.781238555908203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,8192,1,0,29.514401245117188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,10240,2,0,24.48570861816406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,10240,8,0,5.746097564697266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,10240,16,0,2.9423376083374024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,10240,32,0,1.5477727890014648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,10240,64,0,0.9664192199707031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,10240,4,0,11.536444854736327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,10240,2,0,22.954130554199217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,10240,1,0,47.28286437988281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,12288,8,0,9.140316772460938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,12288,4,0,17.03142547607422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,12288,32,0,2.1913583755493162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,12288,64,0,1.2242143630981446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,12288,16,0,4.258257675170898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,10240,1,0,45.606695556640624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,12288,2,0,34.5724365234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,12288,4,0,16.433334350585938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,12288,16,0,4.294268798828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,12288,8,0,8.254395294189454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,12288,64,0,1.3311023712158203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,12288,32,0,2.313649559020996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,12288,2,0,32.67025451660156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,32768,1,0,235.87177734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,16384,16,0,7.516130828857422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,12288,1,0,69.7558349609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,16384,8,0,15.326191711425782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,4,16384,64,0,2.1536287307739257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,16384,4,0,29.9021484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,16384,32,0,3.7770015716552736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,32768,1,0,223.582421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,16384,16,0,7.241236877441406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,16384,32,0,3.836376190185547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,12288,1,0,66.01902465820312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,1,1,0,0.015268799662590028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,1,2,0,0.014319999516010285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,1,4,0,0.014086399972438813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,1,8,0,0.013867199420928955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,16384,8,0,14.682435607910156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,1,16,0,0.01393280029296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,1,64,0,0.014017599821090698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,1,32,0,0.013748799264431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,1,1,0,0.0200080007314682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,1,2,0,0.020497600734233856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,1,4,0,0.01929759979248047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,1,16,0,0.019009600579738616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,1,32,0,0.01903360038995743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,1,64,0,0.019180800020694732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,16,1,0,0.017071999609470367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,16,2,0,0.01656160056591034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,16,4,0,0.01615840047597885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,16,8,0,0.014902399480342865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,16,16,0,0.014798399806022645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,16,32,0,0.013288000226020813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,16,64,0,0.013033600151538849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,16,1,0,0.023214399814605713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,16,2,0,0.022339199483394623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,16,4,0,0.021668800711631776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,16,8,0,0.021427200734615327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,16,16,0,0.019713599979877473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,16,32,0,0.0192208006978035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,16,64,0,0.01902559995651245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,32,1,0,0.02685439884662628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,32,2,0,0.01648160070180893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,32,4,0,0.015468800067901611
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,32,8,0,0.015609599649906158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,32,16,0,0.01539359986782074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,32,32,0,0.013952000439167023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,32,64,0,0.013504000008106231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,32,1,0,0.03584479987621307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,1,8,0,0.020153599977493285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,32,2,0,0.023296000063419343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,32,4,0,0.02195200026035309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,32,16,0,0.020334400236606598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,32,32,0,0.019790400564670563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,32,64,0,0.01920959949493408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,64,1,0,0.04907520115375519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,64,2,0,0.026982399821281432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,64,4,0,0.017283199727535246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,64,8,0,0.01661919951438904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,64,16,0,0.016198399662971496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,64,32,0,0.015332800149917603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,64,64,0,0.014710399508476257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,64,1,0,0.05937759876251221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,64,2,0,0.03630560040473938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,64,4,0,0.023924799263477327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,64,8,0,0.0230320006608963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,64,16,0,0.022491200268268584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,64,32,0,0.021432000398635864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,64,64,0,0.021076799929142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,4,16384,64,0,2.0143775939941406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,128,1,0,0.08911200165748596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,128,2,0,0.053718400001525876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,128,4,0,0.031435200572013856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,128,8,0,0.01964640021324158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,128,32,0,0.01918559968471527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,128,64,0,0.017947199940681457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,128,1,0,0.10043359994888305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,128,2,0,0.06370400190353394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,32,8,0,0.022623999416828154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,128,4,0,0.0428384006023407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,128,8,0,0.02707040011882782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,128,16,0,0.02653439939022064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,128,32,0,0.026495999097824095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,128,64,0,0.025065600872039795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,256,1,0,0.18743679523468018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,256,2,0,0.1045456051826477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,256,4,0,0.06141600012779236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,256,8,0,0.039617601037025454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,256,16,0,0.025832000374794006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,256,32,0,0.02566240131855011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,256,64,0,0.025091201066970825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,128,16,0,0.019227199256420135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,256,2,0,0.11676479578018188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,256,4,0,0.07393280267715455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,256,8,0,0.0535968005657196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,256,16,0,0.03600319921970367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,256,32,0,0.034801599383354184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,256,64,0,0.034200000762939456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,512,1,0,0.45187039375305177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,512,2,0,0.24282240867614746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,16384,4,0,28.801214599609374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,512,4,0,0.13164160251617432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,512,16,0,0.06543840169906616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,512,8,0,0.10504000186920166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,512,32,0,0.04325119853019714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,512,64,0,0.05223199725151062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,16384,2,0,60.676507568359376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,512,2,0,0.28332319259643557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,512,1,0,0.4673664093017578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,512,4,0,0.17574559450149535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,512,8,0,0.12221759557723999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,512,32,0,0.06901760101318359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,256,1,0,0.1991487979888916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,512,64,0,0.07088800072669983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,1024,4,0,0.3811055898666382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,1024,2,0,0.6722911834716797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,1024,16,0,0.1379968047142029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,1024,1,0,1.3485679626464844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,1024,32,0,0.0976639986038208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,1024,64,0,0.07321760058403015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,1024,2,0,0.7311776161193848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,1024,1,0,1.3134079933166505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,1024,4,0,0.382806396484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,1024,16,0,0.16023039817810059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,512,16,0,0.08224959969520569
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,1024,8,0,0.21769919395446777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,1024,32,0,0.12075200080871581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,1024,64,0,0.09570080041885376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,1024,8,0,0.19091039896011353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,1536,8,0,0.3521359920501709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,1536,16,0,0.22668321132659913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,1536,2,0,1.2928959846496582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,1536,32,0,0.15638879537582398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,1536,64,0,0.11613600254058838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,1536,4,0,0.68755202293396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,1536,1,0,2.6507984161376954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,1536,8,0,0.3821104049682617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,1536,2,0,1.3218735694885253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,1536,16,0,0.28056321144104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,1536,32,0,0.18659839630126954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,1536,64,0,0.14939199686050414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,1536,1,0,2.6709503173828124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,1536,4,0,0.6641776084899902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,2048,4,0,1.0649472236633302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,2048,32,0,0.2238464117050171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,2048,8,0,0.5927296161651612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,2048,64,0,0.16572959423065187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,2048,2,0,2.1903312683105467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,2048,4,0,1.1839776039123535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,2048,2,0,2.12390079498291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,2048,8,0,0.6241375923156738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,2048,1,0,4.4793041229248045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,2048,16,0,0.36531360149383546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,2048,32,0,0.2682816028594971
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,2048,64,0,0.21618399620056153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,2048,16,0,0.3135839939117432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,2048,1,0,4.427028656005859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,3072,8,0,1.1280415534973145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,3072,16,0,0.614470386505127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,3072,32,0,0.3877295970916748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,3072,64,0,0.27040159702301025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,3072,2,0,4.624844741821289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,3072,4,0,2.267353630065918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,3072,4,0,2.3294399261474608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,3072,8,0,1.1776415824890136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,3072,16,0,0.683400011062622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,3072,32,0,0.4502384185791016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,3072,64,0,0.3341248035430908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,3072,1,0,9.244918060302734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,3072,2,0,4.677852630615234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,3072,1,0,8.929532623291015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,4096,16,0,1.0151391983032227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,4096,8,0,1.961207962036133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,4096,64,0,0.4217199802398682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,16384,2,0,58.317535400390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,4096,2,0,8.021128082275391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,4096,4,0,3.9785152435302735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,4096,8,0,1.9354639053344727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,4096,4,0,3.8588912963867186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,4096,16,0,1.0994976043701172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,4096,2,0,8.12359848022461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,4096,32,0,0.6415919780731201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,4096,64,0,0.510361623764038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,4096,1,0,15.934194946289063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,4096,32,0,0.5777472019195556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,6144,16,0,2.1411775588989257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,6144,8,0,4.294595336914062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,4096,1,0,15.418525695800781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,6144,64,0,0.7963856220245361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,6144,4,0,8.60533447265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,6144,32,0,1.2013296127319335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,6144,8,0,4.161711883544922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,6144,16,0,2.2775104522705076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,6144,2,0,18.335586547851562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,6144,32,0,1.2299551963806152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,6144,4,0,8.97735824584961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,6144,64,0,0.8733280181884766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,16384,1,0,120.4046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,6144,2,0,16.655026245117188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,8192,16,0,3.782883071899414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,8192,8,0,7.649985504150391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,8192,32,0,2.0046911239624023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,8,8192,64,0,1.1217359542846679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,6144,1,0,35.20885314941406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,8192,4,0,15.539144897460938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,6144,1,0,35.094329833984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,8192,8,0,7.850772857666016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,16384,1,0,114.7651611328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,8192,16,0,3.7824047088623045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,1,1,0,0.01436000019311905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,8192,32,0,2.169379234313965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,1,2,0,0.013961599767208099
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,1,8,0,0.013644799590110779
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,1,16,0,0.01366879940032959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,1,32,0,0.012831999361515046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,1,64,0,0.013683199882507324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,1,1,0,0.0205487996339798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,1,2,0,0.019988800585269927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,8,8192,64,0,1.2254768371582032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,1,4,0,0.019702400267124175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,1,16,0,0.019009600579738616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,1,8,0,0.01857600063085556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,1,32,0,0.019139200448989868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,1,64,0,0.01838400065898895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,16,1,0,0.02975359857082367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,16,2,0,0.016603200137615202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,16,4,0,0.01632159948348999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,16,8,0,0.015830400586128234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,16,16,0,0.014795200526714325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,16,32,0,0.01395999938249588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,16,64,0,0.013769599795341491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,16,1,0,0.03800959885120392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,16,2,0,0.02346239984035492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,16,4,0,0.021913599967956544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,16,8,0,0.022598400712013245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,16,16,0,0.020127999782562255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,16,32,0,0.020766399800777435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,16,64,0,0.019415999948978423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,32,1,0,0.05047360062599182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,32,2,0,0.026950401067733765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,32,4,0,0.017713600397109987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,32,8,0,0.016443200409412384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,32,16,0,0.01704320013523102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,32,32,0,0.014910399913787842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,32,64,0,0.01496800035238266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,32,1,0,0.05924000144004822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,32,2,0,0.037201601266860965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,32,4,0,0.023876799643039702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,32,8,0,0.02396160066127777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,32,16,0,0.022779199481010436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,32,32,0,0.022126400470733644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,32,64,0,0.020351999998092653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,64,1,0,0.08789439797401429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,64,4,0,0.029702401161193846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,64,8,0,0.019182400405406953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,64,16,0,0.01858240067958832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,64,32,0,0.018512000143527985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,8192,2,0,30.53862609863281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,8192,4,0,15.12301025390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,64,64,0,0.017246399819850922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,64,4,0,0.040556800365447995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,64,1,0,0.10836319923400879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,64,16,0,0.026081600785255434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,64,32,0,0.02605920135974884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,64,64,0,0.02452320009469986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,128,1,0,0.1626528024673462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,128,2,0,0.09104800224304199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,128,4,0,0.05577920079231262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,128,8,0,0.034727999567985536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,128,16,0,0.02284799963235855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,128,32,0,0.02274080067873001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,128,64,0,0.02210720032453537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,64,2,0,0.051230400800704956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,128,1,0,0.1755903959274292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,128,2,0,0.10340800285339355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,128,4,0,0.06909440159797668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,128,8,0,0.04832479953765869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,128,16,0,0.032876798510551454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,128,32,0,0.031672000885009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,128,64,0,0.031276801228523256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,64,8,0,0.02632960081100464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,256,1,0,0.35637760162353516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,256,2,0,0.1972383975982666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,256,4,0,0.10667519569396973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,256,8,0,0.06877279877662659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,256,16,0,0.04765760004520416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,256,64,0,0.031585600972175595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,256,1,0,0.3719887971878052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,256,2,0,0.2108464002609253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,1,4,0,0.013806399703025819
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,64,2,0,0.0676144003868103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,256,8,0,0.08522400259971619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,256,16,0,0.06392639875411987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,256,64,0,0.04476479887962341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,256,32,0,0.0324400007724762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,512,1,0,0.8890208244323731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,512,4,0,0.2511967897415161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,8192,2,0,30.234963989257814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,512,8,0,0.17378560304641724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,512,16,0,0.1192639946937561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,512,32,0,0.07932320237159729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,512,64,0,0.058779197931289676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,512,2,0,0.5119279861450196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,512,1,0,0.9237504005432129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,512,4,0,0.2755824089050293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,256,4,0,0.12362240552902222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,256,32,0,0.04771359860897064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,512,8,0,0.19848480224609374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,512,16,0,0.13425439596176147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,512,32,0,0.10315999984741211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,512,64,0,0.08341919779777526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,1024,8,0,0.36849279403686525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,1024,16,0,0.25253920555114745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,1024,4,0,0.6841087818145752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,1024,32,0,0.1684864044189453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,512,2,0,0.4800879955291748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,1024,4,0,0.722047996520996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,1024,2,0,1.3666463851928712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,1024,1,0,2.5784975051879884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,1024,8,0,0.4074079990386963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,1024,16,0,0.29259040355682375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,1024,64,0,0.16782399415969848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,1024,32,0,0.21521439552307128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,1024,1,0,2.603473663330078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,1536,8,0,0.6982927799224854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,1536,4,0,1.3373071670532226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,1024,2,0,1.3280415534973145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,1536,32,0,0.2822191953659058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,1536,16,0,0.4409791946411133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,1536,64,0,0.19756959676742553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,1536,2,0,2.5950944900512694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,1536,8,0,0.7999135971069335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,1536,16,0,0.5091184139251709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,1536,2,0,2.6519071578979494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,1024,64,0,0.12624640464782716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,1536,32,0,0.3461616039276123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,1536,1,0,5.325398254394531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,1536,64,0,0.264355206489563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,2048,8,0,1.119428825378418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,1536,4,0,1.3522751808166504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,2048,16,0,0.6332335948944092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,2048,64,0,0.2893728017807007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,2048,32,0,0.4127984046936035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,2048,2,0,4.528518295288086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,1536,1,0,5.246227264404297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,2048,4,0,2.208679962158203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,2048,4,0,2.204113578796387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,2048,16,0,0.6810192108154297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,2048,2,0,4.4189903259277346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,2048,8,0,1.1666784286499023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,2048,64,0,0.3592592000961304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,8192,1,0,61.1999755859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,2048,32,0,0.5010223865509034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,2048,1,0,9.094737243652343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,3072,16,0,1.2131263732910156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,3072,64,0,0.5063615798950195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,3072,32,0,0.7603807926177979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,3072,8,0,2.456563186645508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,2048,1,0,8.708108520507812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,3072,4,0,4.741966247558594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,3072,16,0,1.4535344123840332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,3072,8,0,2.3428735733032227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,3072,64,0,0.6069632053375245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,3072,32,0,0.8790255546569824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,3072,4,0,4.877505493164063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,3072,2,0,9.416551971435547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,8192,1,0,59.41343383789062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,3072,2,0,9.225105285644531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,4096,16,0,2.0368080139160156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,4096,32,0,1.1258496284484862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,4096,8,0,4.118614578247071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,16,4096,64,0,0.762883186340332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,4096,4,0,8.275421142578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,3072,1,0,18.547314453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,4096,16,0,2.150424003601074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,3072,1,0,18.296917724609376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,4096,4,0,8.01523666381836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,4096,32,0,1.260043239593506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,16,4096,64,0,0.9574015617370606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,1,1,0,0.015027199685573579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,1,2,0,0.015048000216484069
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,1,4,0,0.013844799995422364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,1,8,0,0.01478559970855713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,1,16,0,0.013659200072288514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,1,32,0,0.014684799313545226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,32,1,64,0,0.013505600392818451
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,1,1,0,0.02231519967317581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,1,2,0,0.019171200692653656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,1,4,0,0.02123039960861206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,1,16,0,0.021155199408531188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,1,32,0,0.021135999262332915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,32,1,64,0,0.021264000236988066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,16,1,0,0.050327998399734494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,16,2,0,0.02749119997024536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,16,4,0,0.017918400466442108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,4096,2,0,16.175108337402342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,16,8,0,0.01727840006351471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,16,32,0,0.015281599760055543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,16,16,0,0.01762239933013916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,32,16,64,0,0.015089599788188935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,16,2,0,0.03720479905605316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,16,1,0,0.06340960264205933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,16,4,0,0.024527999758720397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,16,8,0,0.025481599569320678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,16,16,0,0.02292319983243942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,16,32,0,0.02350880056619644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,32,16,64,0,0.021356800198554994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,32,1,0,0.08805760145187377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,32,2,0,0.051502400636672975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,32,8,0,0.01923519968986511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,32,16,0,0.018580800294876097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,32,32,0,0.018067200481891633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,32,32,64,0,0.016811199486255646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,32,1,0,0.09735519886016845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,32,2,0,0.06212639808654785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,32,4,0,0.04088160097599029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,32,8,0,0.026174399256706237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,1,8,0,0.01884640008211136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,32,16,0,0.025440001487731935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,32,32,0,0.02505599856376648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,32,32,64,0,0.02396479994058609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,64,2,0,0.08977439999580383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,64,4,0,0.05491999983787536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,64,8,0,0.03412480056285858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,64,16,0,0.022312000393867493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,64,32,0,0.02178719937801361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,32,64,64,0,0.021756799519062044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,64,1,0,0.16719839572906495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,64,2,0,0.10292479991912842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,64,4,0,0.0675055980682373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,64,8,0,0.04698559939861298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,64,16,0,0.032918399572372435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,32,4,0,0.029919999837875366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,4096,2,0,15.510873413085937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,64,32,0,0.03129599988460541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,32,64,64,0,0.03099200129508972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,128,4,0,0.09825279712677001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,128,8,0,0.06300320029258728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,128,1,0,0.35055038928985593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,128,16,0,0.0425247997045517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,128,32,0,0.03140799999237061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,32,128,64,0,0.029049599170684816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,128,2,0,0.18192000389099122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,128,1,0,0.35275518894195557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,128,4,0,0.11476160287857055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,128,8,0,0.08093760013580323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,128,16,0,0.06006079912185669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,64,1,0,0.15740159749984742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,32,128,64,0,0.04181120097637177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,256,2,0,0.37874879837036135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,256,1,0,0.6987775802612305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,4096,8,0,4.124937438964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,256,8,0,0.12012159824371338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,256,16,0,0.08202400207519531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,256,32,0,0.06062080264091492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,32,256,64,0,0.046312001347541806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,256,2,0,0.40428638458251953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,128,2,0,0.1655791997909546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,256,1,0,0.725278377532959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,256,8,0,0.14539680480957032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,256,4,0,0.22453761100769043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,256,16,0,0.10638400316238403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,256,32,0,0.08576480150222779
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,512,4,0,0.49550881385803225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,512,2,0,0.9479663848876954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,128,32,0,0.04599519968032837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,512,8,0,0.3296303987503052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,512,32,0,0.13829599618911742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,32,512,64,0,0.10744479894638062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,512,1,0,1.7716575622558595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,256,4,0,0.19855040311813354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,512,4,0,0.5362751960754395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,512,2,0,0.9913904190063476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,512,8,0,0.3690639972686768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,512,32,0,0.18196320533752441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,512,16,0,0.24359519481658937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,512,1,0,1.7937023162841796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,32,256,64,0,0.06901440024375916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,1024,4,0,1.3466400146484374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,512,16,0,0.2012495994567871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,1024,16,0,0.47655677795410156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,1024,8,0,0.7458799839019775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,1024,2,0,2.726318359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,4096,1,0,33.162603759765624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,1024,32,0,0.30794239044189453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,32,512,64,0,0.1492687940597534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,1024,8,0,0.7994383811950684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,1024,4,0,1.4207584381103515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,1024,1,0,5.288321685791016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,1024,16,0,0.5581039905548095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,1024,32,0,0.3895375967025757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,32,1024,64,0,0.3138191938400269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,1024,2,0,2.7575872421264647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,32,1024,64,0,0.22745440006256104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,1536,8,0,1.359761619567871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,1536,16,0,0.8495408058166504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,4096,1,0,32.36716918945312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,1024,1,0,5.2380718231201175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,32,1536,64,0,0.3628880023956299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,1536,4,0,2.6762943267822266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,1536,8,0,1.5385631561279296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,1536,2,0,5.464739227294922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,1536,16,0,0.9532959938049317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,1536,4,0,2.7045263290405273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,1536,32,0,0.6737808227539063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,32,1536,64,0,0.4938655853271484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,1536,32,0,0.5234975814819336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,2048,16,0,1.2065216064453126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,1536,2,0,5.27167854309082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,2048,8,0,2.2684383392333984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,32,2048,64,0,0.5239408016204834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,2048,32,0,0.7877488136291504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,1536,1,0,10.775875091552734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,2048,4,0,4.458583831787109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,2048,16,0,1.3990511894226074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,2048,8,0,2.3343488693237306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,2048,32,0,0.9199695587158203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,1536,1,0,10.808419036865235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,1,1,0,0.015030400454998016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,1,4,0,0.014046399295330048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,1,8,0,0.013711999356746673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,1,16,0,0.013731199502944946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,32,2048,64,0,0.7041247844696045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,1,32,0,0.013726399838924408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,64,1,64,0,0.014633600413799287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,1,2,0,0.021334399282932282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,2048,2,0,8.920540618896485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,1,4,0,0.020793600380420683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,1,16,0,0.02067199945449829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,1,8,0,0.021513600647449494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,1,32,0,0.020632000267505647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,64,1,64,0,0.021529600024223328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,16,1,0,0.09081439971923828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,16,2,0,0.0553056001663208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,16,4,0,0.031134399771690368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,16,8,0,0.02149759978055954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,16,16,0,0.019424000382423402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,64,16,64,0,0.01823039948940277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,16,1,0,0.10119999647140503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,16,2,0,0.06507999897003174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,16,4,0,0.04196160137653351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,16,8,0,0.028068798780441283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,16,16,0,0.026440000534057616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,16,32,0,0.025016000866889952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,64,16,64,0,0.02452480047941208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,2048,4,0,4.49658088684082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,32,1,0,0.1578976035118103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,1,2,0,0.015160000324249268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,32,4,0,0.05619360208511352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,32,2,0,0.09456800222396851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,32,8,0,0.03540799915790558
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,32,32,0,0.022035199403762817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,64,32,64,0,0.020695999264717102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,1,1,0,0.02160799950361252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,32,1,0,0.16705440282821654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,32,2,0,0.10390399694442749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,32,4,0,0.06923999786376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,32,16,0,0.03315840065479279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,32,32,0,0.031832000613212584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,64,32,64,0,0.030721598863601686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,64,1,0,0.29528160095214845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,64,2,0,0.16263680458068847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,64,8,0,0.06234719753265381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,64,16,0,0.04270879924297333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,16,32,0,0.01786559969186783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,64,64,64,0,0.029028800129890443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,64,1,0,0.3091743946075439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,64,2,0,0.17659519910812377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,64,4,0,0.1133072018623352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,64,8,0,0.07796000242233277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,64,16,0,0.05867999792098999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,64,32,0,0.04371039867401123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,64,64,64,0,0.04219039976596832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,2048,2,0,8.941881561279297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,32,8,0,0.048230400681495665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,128,2,0,0.33215041160583497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,128,1,0,0.6068480014801025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,128,4,0,0.18055360317230223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,128,8,0,0.11225759983062744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,128,16,0,0.07695199847221375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,128,32,0,0.05660960078239441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,2048,1,0,17.71215362548828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,64,4,0,0.09899839758872986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,64,32,0,0.029718399047851562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,128,4,0,0.20575358867645263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,128,8,0,0.13786239624023439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,128,16,0,0.10220320224761963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,128,1,0,0.6337279796600341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,128,32,0,0.08171839714050293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,64,128,64,0,0.06653119921684265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,256,4,0,0.38423359394073486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,256,8,0,0.22727200984954835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,256,16,0,0.14903839826583862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,32,16,0,0.023665599524974823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,256,32,0,0.11078399419784546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,256,1,0,1.3829392433166503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,64,256,64,0,0.08372319936752319
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,2048,1,0,17.353924560546876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,256,4,0,0.4292304039001465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,256,2,0,0.7988768100738526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,256,16,0,0.19327839612960815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,64,128,64,0,0.04427359998226166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,256,8,0,0.2822144031524658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,64,256,64,0,0.12355680465698242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,256,32,0,0.15494560003280639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,256,1,0,1.4210351943969726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,128,2,0,0.3733455896377563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,512,16,0,0.38260319232940676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,512,8,0,0.6414688110351563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,512,32,0,0.2541311979293823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,512,4,0,0.9702816009521484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,64,512,64,0,0.19383840560913085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,256,2,0,0.7507376194000244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,512,2,0,1.891334342956543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,512,16,0,0.4626192092895508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,512,4,0,1.0511728286743165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,512,32,0,0.3457904100418091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,64,512,64,0,0.27608160972595214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,512,2,0,1.9393007278442382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,512,1,0,3.532571029663086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,1024,16,0,0.9253664016723633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,512,1,0,3.621160125732422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,1024,8,0,1.4096495628356933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,64,1024,64,0,0.4168848037719727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,1024,32,0,0.5760047912597657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,1024,4,0,2.7542320251464845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,512,8,0,0.7144527912139893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,1024,32,0,0.7384960174560546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,1024,16,0,1.157148838043213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,1024,8,0,1.5644783973693848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,1,1,0,0.017817600071430205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,1,2,0,0.01579039990901947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,1,4,0,0.015881599485874177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,64,1024,64,0,0.5666800022125245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,1,8,0,0.015731200575828552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,128,1,64,0,0.015747199952602386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,1,32,0,0.015161600708961488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,1024,2,0,5.850067138671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,1,1,0,0.023894399404525757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,1024,4,0,2.8346656799316405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,1,8,0,0.021408000588417055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,1,4,0,0.023875199258327484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,1,32,0,0.021751999855041504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,1,16,0,0.0226160004734993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,128,1,64,0,0.023766399919986726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,16,1,0,0.15640000104904175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,16,2,0,0.09477440118789673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,16,8,0,0.03579199910163879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,16,32,0,0.02181600034236908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,128,16,64,0,0.022060799598693847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,16,1,0,0.16772160530090333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,16,2,0,0.10385279655456543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,16,4,0,0.06884160041809081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,16,8,0,0.049039998650550844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,1024,2,0,5.498219299316406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,16,16,0,0.033564800024032594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,128,16,64,0,0.03151200115680695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,1,16,0,0.015964800119400026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,32,2,0,0.1629807949066162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,32,1,0,0.2943536043167114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,1,2,0,0.021796800196170807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,32,8,0,0.06309599876403808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,32,16,0,0.04299040138721466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,128,32,64,0,0.028585600852966308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,1024,1,0,10.754563140869141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,32,1,0,0.31488800048828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,16,4,0,0.05773280262947082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,32,4,0,0.11521439552307129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,32,8,0,0.07905439734458923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,32,16,0,0.059680002927780154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,32,32,0,0.04451839923858643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,128,32,64,0,0.042340800166130066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,64,2,0,0.30963840484619143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,64,1,0,0.5752384185791015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,64,4,0,0.1773327946662903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,64,8,0,0.11155359745025635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,64,16,0,0.07515680193901061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,64,32,0,0.05515199899673462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,128,64,64,0,0.04471679925918579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,16,32,0,0.03269439935684204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,1024,1,0,10.47216796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,32,4,0,0.09966719746589661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,64,2,0,0.33418879508972166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,32,32,0,0.02985599935054779
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,64,8,0,0.13715519905090331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,64,16,0,0.10106559991836547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,64,32,0,0.08013280034065247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,128,64,64,0,0.06982240080833435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,128,4,0,0.3476144075393677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,32,2,0,0.18963999748229982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,128,2,0,0.6196815967559814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,128,8,0,0.2104543924331665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,128,16,0,0.14112639427185059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,128,32,0,0.10569599866867066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,128,128,64,0,0.08022400140762329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,128,1,0,1.203171157836914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,128,4,0,0.39079999923706055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,128,8,0,0.25294880867004393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,128,2,0,0.6617983818054199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,128,16,0,0.18591680526733398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,128,32,0,0.14878720045089722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,128,128,64,0,0.12001600265502929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,128,1,0,1.243931198120117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,256,8,0,0.4352848052978516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,256,16,0,0.2797391891479492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,256,2,0,1.4768079757690429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,256,32,0,0.20314240455627441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,128,256,64,0,0.1572432041168213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,16,16,0,0.02431199997663498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,256,1,0,2.7650096893310545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,256,4,0,0.8297904014587403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,256,16,0,0.36153440475463866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,256,8,0,0.5241519927978515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,256,32,0,0.2838831901550293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,256,4,0,0.749512004852295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,256,1,0,2.837763214111328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,512,4,0,1.936180877685547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,512,8,0,1.2444144248962403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,256,2,0,1.578752040863037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,128,256,64,0,0.23713600635528564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,512,32,0,0.48919038772583007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,128,512,64,0,0.360536003112793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,64,1,0,0.6034255981445312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,512,2,0,3.7539806365966797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,512,16,0,0.886787223815918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,512,4,0,2.0583839416503906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,512,32,0,0.6344624042510987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,128,512,64,0,0.5150464057922364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,1,1,0,0.03132160007953644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,512,1,0,7.067508697509766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,1,2,0,0.023608000576496126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,1,4,0,0.022355200350284578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,512,16,0,0.7487631797790527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,64,4,0,0.2135551929473877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,1,32,0,0.02197760045528412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,1,8,0,0.022473600506782532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,256,1,64,0,0.022579200565814972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,1,2,0,0.02884480059146881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,1,1,0,0.033851200342178346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,1,4,0,0.02924000024795532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,512,2,0,3.908465576171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,1,16,0,0.028830400109291075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,1,32,0,0.028603199124336242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,256,1,64,0,0.028276801109313965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,16,4,0,0.10052160024642945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,16,2,0,0.16744159460067748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,16,1,0,0.3098223924636841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,16,16,0,0.042633599042892455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,16,32,0,0.03578400015830994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,256,16,64,0,0.03478559851646423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,16,2,0,0.18265919685363768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,16,4,0,0.11508320569992066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,512,8,0,1.3943504333496093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,16,8,0,0.08116639852523803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,256,16,64,0,0.04865280091762543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,16,32,0,0.052883201837539674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,32,2,0,0.30964319705963134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,32,1,0,0.5891791820526123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,32,4,0,0.17827359437942505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,32,8,0,0.11356960535049439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,512,1,0,7.192289733886719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,32,32,0,0.05536320209503174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,1,16,0,0.022163200378417968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,256,32,64,0,0.05135679841041565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,1,8,0,0.029185599088668822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,32,2,0,0.3360975980758667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,32,8,0,0.13908640146255494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,32,1,0,0.6031824111938476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,32,4,0,0.20763039588928223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,32,16,0,0.1018831968307495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,32,32,0,0.08232640027999878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,256,32,64,0,0.07529119849205017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,64,8,0,0.20952320098876953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,16,1,0,0.312337589263916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,64,4,0,0.3442863941192627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,64,2,0,0.6097936153411865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,16,16,0,0.05951200127601623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,256,64,64,0,0.08205919861793518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,64,32,0,0.10511519908905029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,64,1,0,1.1394271850585938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,64,4,0,0.38387041091918944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,64,2,0,0.6525023937225342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,64,8,0,0.2508368015289307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,64,16,0,0.1859055995941162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,256,64,64,0,0.12643200159072876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,64,32,0,0.148198401927948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,64,1,0,1.1850799560546874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,32,16,0,0.07661759853363037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,128,16,0,0.2639791965484619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,128,8,0,0.4007823944091797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,128,4,0,0.6715439796447754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,128,32,0,0.1961840033531189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,256,128,64,0,0.15232160091400146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,16,8,0,0.06372640132904053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,128,2,0,1.2191632270812989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,128,8,0,0.48297758102416993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,128,16,0,0.34361441135406495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,64,16,0,0.1432144045829773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,128,1,0,2.3857311248779296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,128,2,0,1.2989007949829101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,256,128,64,0,0.23383519649505616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,128,1,0,2.4648464202880858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,256,8,0,0.8492575645446777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,256,4,0,1.4754799842834472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,1,256,256,64,0,0.2935760021209717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,256,32,0,0.38856959342956543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,256,2,0,2.9326736450195314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,256,16,0,0.5381919860839843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,128,32,0,0.2777008056640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,256,8,0,0.9977968215942383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,256,16,0,0.6941120147705078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,256,4,0,1.6167295455932618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,256,1,0,5.454937744140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,256,32,0,0.5421328067779541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,256,2,0,3.123531150817871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,1,1,0,0.013174399733543396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,1,2,0,0.012555199861526489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,128,4,0,0.7514895915985107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,1,32,0,0.012057600170373916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,1,8,0,0.012352000176906585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,1,256,256,64,0,0.45130081176757814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,1,4,0,0.012201599776744843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,1,4,0,0.018507200479507446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,1,16,0,0.01767520010471344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,1,2,0,0.018667200207710268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,1,16,0,0.012217599898576736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,1,64,0,0.017367999255657195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,1,1,0,0.01918559968471527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,256,1,0,5.634172821044922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,16,2,0,0.013969600200653076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,16,8,0,0.013808000087738036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,16,32,0,0.013468800485134125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,16,64,0,0.013065600395202636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,16,1,0,0.021505600214004515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,16,16,0,0.013436800241470337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,16,4,0,0.01987999975681305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,16,16,0,0.01974080055952072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,16,32,0,0.019313600659370423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,1,64,0,0.0118928000330925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,16,64,0,0.018721599876880646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,32,1,0,0.015139199793338776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,32,2,0,0.015358400344848634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,32,4,0,0.013955199718475341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,32,16,0,0.013672000169754029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,1,32,0,0.018006399273872375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,32,64,0,0.013515199720859527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,32,1,0,0.022339199483394623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,32,2,0,0.02147199958562851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,32,4,0,0.020278400182723998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,16,4,0,0.013497599959373474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,32,8,0,0.019726400077342988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,32,16,0,0.019776000082492827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,32,32,0,0.0195360004901886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,32,64,0,0.019276799261569978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,64,1,0,0.01627040058374405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,64,2,0,0.015408000349998474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,64,4,0,0.015369600057601929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,64,8,0,0.014159999787807465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,64,16,0,0.014023999869823455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,64,32,0,0.013827200233936309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,64,64,0,0.013867199420928955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,64,1,0,0.022859199345111846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,64,2,0,0.022392000257968902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,16,2,0,0.02027360051870346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,16,8,0,0.01903039962053299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,64,16,0,0.020131200551986694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,64,8,0,0.02004159986972809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,64,64,0,0.0199072003364563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,128,1,0,0.030199998617172243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,128,2,0,0.017815999686717987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,128,4,0,0.017183999717235564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,128,8,0,0.016598400473594666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,128,16,0,0.015692800283432007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,128,32,0,0.015619200468063355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,128,64,0,0.015564799308776855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,128,1,0,0.040222400426864625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,128,2,0,0.0241007998585701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,128,4,0,0.023924799263477327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,128,8,0,0.02332320064306259
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,128,16,0,0.022654399275779724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,128,32,0,0.021731199324131013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,128,64,0,0.021505600214004515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,256,1,0,0.05845280289649964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,32,32,0,0.013662399351596832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,256,4,0,0.020623999834060668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,256,8,0,0.01977120041847229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,256,16,0,0.018960000574588777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,256,32,0,0.019012799859046935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,256,64,0,0.018862399458885192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,256,1,0,0.06823520064353943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,256,2,0,0.03857919871807099
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,256,4,0,0.027267199754714967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,256,8,0,0.026817598938941957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,256,16,0,0.025646400451660157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,256,32,0,0.025155198574066163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,64,4,0,0.021324799954891206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,256,64,0,0.02497600018978119
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,64,32,0,0.01915840059518814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,512,2,0,0.06985920071601867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,512,4,0,0.039473599195480345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,512,8,0,0.02696320116519928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,512,16,0,0.03643999993801117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,512,32,0,0.028248000144958495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,512,64,0,0.02402079999446869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,512,1,0,0.13610719442367553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,512,2,0,0.07980639934539795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,512,4,0,0.04764159917831421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,512,8,0,0.03373279869556427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,512,16,0,0.0446399986743927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,512,64,0,0.03080480098724365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,32,8,0,0.013990400731563568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,256,2,0,0.030131199955940248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,1024,1,0,0.3296191930770874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,16,1,0,0.015185600519180298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,1024,2,0,0.1786720037460327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,1024,8,0,0.05073760151863098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,1024,16,0,0.04092479944229126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,1024,4,0,0.09570559859275818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,1024,64,0,0.03188000023365021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,1024,4,0,0.10421760082244873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,1024,2,0,0.18748159408569337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,1024,1,0,0.3391792058944702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,1024,8,0,0.062489598989486694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,1024,16,0,0.04885280132293701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,1024,32,0,0.05328959822654724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,512,1,0,0.1270095944404602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,1536,4,0,0.17548160552978515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,1536,2,0,0.33044960498809817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,1536,8,0,0.09687680006027222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,1536,1,0,0.6231760025024414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,1536,32,0,0.061755198240280154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,1536,64,0,0.0429280012845993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,512,32,0,0.035235199332237246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,1536,1,0,0.6334335803985596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,1536,4,0,0.1840880036354065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,1536,2,0,0.33997440338134766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,1,8,0,0.018174399435520173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,1536,8,0,0.10741440057754517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,1536,32,0,0.07177760004997254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,1536,64,0,0.056068801879882814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,2048,4,0,0.2778847932815552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,2048,16,0,0.07859839797019959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,2048,32,0,0.06837760210037232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,1024,32,0,0.04314239919185638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,2048,1,0,1.0095616340637208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,2048,64,0,0.059462398290634155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,2048,4,0,0.28741281032562255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,2048,2,0,0.5560912132263184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,1024,64,0,0.0392304003238678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,2048,8,0,0.15625760555267335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,2048,16,0,0.0924560010433197
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,2048,64,0,0.07229599952697754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,2048,1,0,1.014350414276123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,1536,16,0,0.05499039888381958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,3072,8,0,0.2830319881439209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,3072,4,0,0.5652671813964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,3072,2,0,1.071059226989746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,1536,16,0,0.06549280285835266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,3072,32,0,0.0982208013534546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,3072,64,0,0.08590880036354065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,3072,1,0,2.1544368743896483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,2048,2,0,0.5362095832824707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,3072,16,0,0.1781167984008789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,3072,4,0,0.5862063884735107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,3072,8,0,0.3326800107955933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,3072,32,0,0.11214079856872558
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,3072,64,0,0.09683359861373901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,3072,2,0,1.069491195678711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,2048,32,0,0.07851200103759766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,4096,8,0,0.48578081130981443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,3072,1,0,2.0792495727539064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,4096,4,0,0.9306703567504883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,4096,32,0,0.1381983995437622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,4096,64,0,0.13045120239257812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,4096,2,0,1.8317472457885742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,3072,16,0,0.16140639781951904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,4096,4,0,0.9447039604187012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,2048,8,0,0.14544320106506348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,4096,2,0,1.8693056106567383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,4096,32,0,0.14948480129241942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,4096,16,0,0.2600320100784302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,4096,64,0,0.15038080215454103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,4096,8,0,0.5050960063934327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,4096,16,0,0.24609439373016356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,6144,8,0,1.015828800201416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,4096,1,0,3.826655960083008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,6144,16,0,0.5081247806549072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,6144,32,0,0.31943678855895996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,6144,64,0,0.20150880813598632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,6144,4,0,2.0861711502075195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,4096,1,0,3.8871696472167967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,6144,2,0,4.277479934692383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,6144,8,0,1.0363776206970214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,6144,32,0,0.34906558990478515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,6144,64,0,0.21100480556488038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,6144,4,0,2.003236770629883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,6144,2,0,3.9755695343017576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,6144,1,0,8.656031799316406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,8192,16,0,0.8796208381652832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,8192,32,0,0.4852575778961182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,8192,64,0,0.24607200622558595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,8192,8,0,1.8114879608154297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,8192,4,0,3.6376529693603517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,6144,1,0,8.071489715576172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,6144,16,0,0.5168399810791016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,8192,8,0,1.845088005065918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,8192,64,0,0.283841609954834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,8192,32,0,0.4752607822418213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,8192,2,0,7.62156982421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,8192,4,0,3.5634078979492188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,8192,16,0,0.8701519966125488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,10240,8,0,2.834337615966797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,10240,32,0,0.742251205444336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,10240,64,0,0.41402239799499513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,10240,16,0,1.3411151885986328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,10240,4,0,5.785849761962891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,8192,1,0,14.817921447753907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,8192,2,0,6.985660552978516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,10240,16,0,1.3167216300964355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,10240,8,0,2.732807922363281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,8192,1,0,14.420318603515625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,10240,64,0,0.44109277725219725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,10240,32,0,0.7148079872131348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,10240,4,0,5.588908767700195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,10240,2,0,11.57436294555664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,12288,8,0,4.303579330444336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,10240,2,0,11.099517059326171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,12288,16,0,1.9261392593383788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,12288,64,0,0.5466432094573974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,12288,32,0,1.149550437927246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,12288,4,0,8.196665954589843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,10240,1,0,23.260151672363282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,12288,8,0,3.986288070678711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,12288,32,0,1.0368335723876954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,12288,16,0,2.1584768295288086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,12288,64,0,0.5993951797485352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,12288,4,0,8.610830688476563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,10240,1,0,23.234909057617188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,12288,2,0,16.769996643066406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,16384,16,0,3.7144737243652344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,12288,2,0,15.850738525390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,16384,32,0,1.8687215805053712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,16384,64,0,0.9013327598571778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,16384,8,0,7.562782287597656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,16384,4,0,15.092398071289063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,12288,1,0,34.232391357421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,12288,1,0,31.994949340820312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,16384,8,0,7.0659950256347654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,16384,16,0,3.446820831298828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,16384,4,0,14.183657836914062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,16384,64,0,0.8932239532470703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,16384,32,0,1.739299201965332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,16384,2,0,30.591549682617188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,16384,2,0,28.331475830078126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,1,32768,16,0,14.33410186767578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,1,32768,32,0,7.122780609130859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,1,32768,64,0,3.609336090087891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,1,32768,8,0,28.638101196289064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,16384,1,0,60.07154541015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,16384,1,0,57.34168701171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,1,32768,16,0,14.176968383789063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,1,32768,32,0,7.064266967773437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,1,32768,4,0,60.08839111328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,1,1,0,0.015230399370193482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,1,2,0,0.014767999947071075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,1,4,0,0.014353600144386292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,1,8,0,0.01403840035200119
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,1,32768,8,0,29.229693603515624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,1,16,0,0.01393280029296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,1,64,0,0.013952000439167023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,1,1,0,0.021267199516296388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,1,2,0,0.020718400180339814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,1,4,0,0.020203199982643128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,1,8,0,0.020113599300384522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,1,16,0,0.01988160014152527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,1,32,0,0.01984799951314926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,1,64,0,0.017836800217628478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,16,1,0,0.015750400722026825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,1,32768,64,0,3.542448043823242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,16,2,0,0.016209599375724793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,16,4,0,0.0160863995552063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,16,8,0,0.015356799960136414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,16,16,0,0.01451680064201355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,16,32,0,0.013991999626159667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,16,64,0,0.013504000008106231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,16,1,0,0.02244639992713928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,16,2,0,0.021719999611377716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,1,32,0,0.013571199774742127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,16,4,0,0.020212799310684204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,16,8,0,0.019489599764347075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,16,16,0,0.01991039961576462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,16,32,0,0.019032000005245207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,16,64,0,0.018884800374507904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,32,1,0,0.015988799929618835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,32,2,0,0.01579039990901947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,32,4,0,0.015497599542140961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,32,8,0,0.014812800288200378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,32,16,0,0.013729600608348847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,32,32,0,0.014004799723625182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,32,64,0,0.013361600041389466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,32,1,0,0.02311680018901825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,32,2,0,0.02213920056819916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,32,4,0,0.021247999370098115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,32,8,0,0.019788800179958342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,32,16,0,0.019699199497699736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,32,32,0,0.019547200202941893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,32,64,0,0.01920959949493408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,64,1,0,0.02512640058994293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,64,2,0,0.016414399445056915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,64,4,0,0.015137599408626556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,64,8,0,0.015535999834537507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,64,16,0,0.014052799344062806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,64,64,0,0.013836799561977387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,64,1,0,0.03351840078830719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,64,2,0,0.02264160066843033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,64,4,0,0.022086399793624877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,64,8,0,0.021201600134372712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,64,16,0,0.02112320065498352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,64,32,0,0.01987680047750473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,64,64,0,0.019625599682331085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,128,1,0,0.051601600646972653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,128,2,0,0.02727999985218048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,128,4,0,0.01770240068435669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,128,8,0,0.017265599966049195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,128,16,0,0.01685439944267273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,128,32,0,0.015945599973201753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,128,64,0,0.015769599378108977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,128,1,0,0.06133120059967041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,128,2,0,0.035120001435279845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,128,4,0,0.024404799938201903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,128,8,0,0.023454399406909944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,128,16,0,0.022593599557876588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,128,32,0,0.022784000635147093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,128,64,0,0.021753600239753722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,256,1,0,0.10091040134429932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,256,2,0,0.05723519921302796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,64,32,0,0.01388320028781891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,256,4,0,0.03146879971027374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,256,8,0,0.021334399282932282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,256,16,0,0.020670400559902193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,256,32,0,0.020467199385166168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,256,64,0,0.019788800179958342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,256,1,0,0.1092352032661438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,256,2,0,0.06733760237693787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,256,4,0,0.040336000919342044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,256,8,0,0.02810879945755005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,256,16,0,0.02747359871864319
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,256,32,0,0.02709600031375885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,256,64,0,0.026305601000785828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,512,2,0,0.12647039890289308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,512,4,0,0.07015680074691773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,512,8,0,0.038940799236297605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,512,16,0,0.03364000022411347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,512,32,0,0.03757280111312866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,512,64,0,0.030132800340652466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,512,1,0,0.24059839248657228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,512,2,0,0.1367184042930603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,512,4,0,0.08030400276184083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,512,8,0,0.04927360117435455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,512,16,0,0.041315200924873355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,512,32,0,0.048230400681495665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,512,64,0,0.03761120140552521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,1024,2,0,0.335265588760376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,1024,1,0,0.6476640224456787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,1024,4,0,0.17678240537643433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,1024,8,0,0.09678559899330139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,1024,16,0,0.05463520288467407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,1024,32,0,0.049003198742866516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,1024,64,0,0.04783360064029694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,512,1,0,0.2310175895690918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,1024,4,0,0.1896656036376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,1024,2,0,0.34939520359039306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,1024,8,0,0.1080288052558899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,1024,16,0,0.06789119839668274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,1024,1,0,0.6582672119140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,1024,32,0,0.05961120128631592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,1536,2,0,0.6499680042266845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,1536,4,0,0.33641600608825684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,1536,1,0,1.2359408378601073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,1536,8,0,0.17726880311965942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,1536,16,0,0.1026960015296936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,1536,32,0,0.0720192015171051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,1536,64,0,0.07832319736480713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,1024,64,0,0.06072319746017456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,1536,2,0,0.6550943851470947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,1536,4,0,0.34540960788726804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,1536,8,0,0.1878880023956299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,1536,16,0,0.11243360042572022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,1536,32,0,0.0811951994895935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,1536,64,0,0.08793280124664307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,1536,1,0,1.2656975746154786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,2048,1,0,2.081977653503418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,2048,2,0,1.065500831604004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,2048,4,0,0.5746431827545166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,2048,8,0,0.29117279052734374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,2048,16,0,0.15733760595321655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,2048,32,0,0.08852319717407227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,2048,64,0,0.08441600203514099
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,2048,4,0,0.5696288108825683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,2048,2,0,1.0650383949279785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,2048,8,0,0.2949167966842651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,1,32768,4,0,56.70218505859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,2048,16,0,0.16465760469436647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,2048,32,0,0.10162080526351928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,2048,64,0,0.10396640300750733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,2048,1,0,2.0234079360961914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,3072,8,0,0.5798704147338867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,3072,4,0,1.120753574371338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,3072,16,0,0.30656321048736573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,3072,32,0,0.17256319522857666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,3072,64,0,0.12086559534072876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,3072,2,0,2.209079933166504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,3072,4,0,1.1999135971069337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,3072,2,0,2.2461727142333983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,3072,8,0,0.602233600616455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,3072,32,0,0.19900000095367432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,3072,16,0,0.3365312099456787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,3072,64,0,0.14162880182266235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,3072,1,0,4.360699081420899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,3072,1,0,4.523009490966797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,4096,4,0,1.9744384765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,4096,2,0,3.939668655395508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,4096,8,0,0.9383968353271485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,4096,16,0,0.5367631912231445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,4096,32,0,0.28960959911346434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,4096,64,0,0.14863040447235107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,4096,4,0,1.8327680587768556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,4096,8,0,1.0022656440734863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,4096,1,0,7.974419403076172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,4096,16,0,0.5324143886566162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,4096,2,0,3.8514801025390626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,4096,32,0,0.3007263898849487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,4096,64,0,0.17486879825592042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,4096,1,0,7.641729736328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,6144,4,0,4.16939697265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,6144,16,0,1.1545488357543945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,6144,8,0,2.161939239501953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,6144,64,0,0.3463295936584473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,6144,32,0,0.5683728218078613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,6144,2,0,8.650873565673828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,6144,4,0,4.158484649658203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,1,32768,2,0,118.57177734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,6144,2,0,8.357376098632812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,6144,8,0,2.254047966003418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,6144,16,0,1.1120479583740235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,6144,32,0,0.6015488147735596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,6144,64,0,0.40635361671447756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,6144,1,0,17.391993713378906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,8192,8,0,4.035220718383789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,8192,16,0,2.0207040786743162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,6144,1,0,17.26128692626953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,8192,4,0,8.256851196289062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,8192,64,0,0.5288496017456055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,8192,32,0,0.9814528465270996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,8192,2,0,16.153361511230468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,1,32768,2,0,112.89150390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,8192,16,0,1.8025968551635743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,8192,32,0,1.011740779876709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,8192,8,0,3.8603824615478515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,8192,4,0,7.439670562744141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,8192,64,0,0.5478544235229492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,8192,2,0,14.686941528320313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,10240,8,0,5.922265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,8192,1,0,30.859759521484374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,10240,32,0,1.4395199775695802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,10240,16,0,3.049028778076172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,10240,4,0,12.427579498291015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,10240,64,0,0.8085136413574219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,8192,1,0,29.704605102539062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,10240,2,0,22.794413757324218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,10240,8,0,6.030487823486328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,10240,4,0,11.705753326416016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,10240,64,0,0.8604319572448731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,10240,32,0,1.590609645843506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,10240,16,0,2.7811552047729493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,10240,2,0,22.74263916015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,12288,8,0,8.710342407226562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,12288,4,0,18.635360717773438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,10240,1,0,48.16862182617187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,12288,16,0,4.5400848388671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,12288,32,0,2.134566307067871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,12288,64,0,1.161019229888916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,10240,1,0,45.64806213378906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,12288,2,0,34.615045166015626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,12288,16,0,4.167633438110352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,12288,8,0,8.237065887451172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,12288,32,0,2.123646354675293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,12288,64,0,1.1308671951293945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,12288,4,0,15.98546142578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,12288,2,0,32.59527587890625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,1,32768,1,0,236.791357421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,12288,1,0,68.51356201171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,16384,32,0,3.8148094177246095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,16384,64,0,1.9208911895751952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,16384,4,0,31.354434204101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,16384,16,0,7.478091430664063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,16384,8,0,15.22806396484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,1,32768,1,0,224.3952392578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,16384,16,0,7.1905517578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,12288,1,0,66.38167724609374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,16384,8,0,14.202615356445312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,16384,32,0,3.6725086212158202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,16384,64,0,1.8435680389404296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,16384,2,0,60.117279052734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,16384,4,0,29.075811767578124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,16384,2,0,57.257147216796874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,2,32768,16,0,29.995208740234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,2,32768,64,0,7.543121337890625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,2,32768,32,0,15.523141479492187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,2,32768,8,0,59.46708984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,16384,1,0,122.00419921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,16384,1,0,115.93790283203126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,2,32768,4,0,119.860498046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,2,32768,16,0,28.884292602539062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,2,32768,8,0,58.255078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,1,1,0,0.01669439971446991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,1,2,0,0.015105600655078887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,1,4,0,0.013415999710559845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,1,8,0,0.013124799728393555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,1,16,0,0.012956799566745758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,1,32,0,0.012982399761676788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,1,64,0,0.012910400331020356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,1,1,0,0.020084799826145174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,1,2,0,0.019415999948978423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,1,4,0,0.019100800156593323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,1,8,0,0.019047999382019044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,1,16,0,0.01876160055398941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,1,32,0,0.018611200153827667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,1,64,0,0.018662400543689728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,16,1,0,0.016990399360656737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,16,2,0,0.015544000267982482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,16,4,0,0.01555519998073578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,16,8,0,0.015113599598407745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,16,16,0,0.013902400434017182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,16,32,0,0.013529600203037262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,16,64,0,0.013203200697898865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,16,1,0,0.023071999847888946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,16,2,0,0.022473600506782532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,16,4,0,0.021649600565433504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,16,8,0,0.020190399885177613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,16,16,0,0.01995519995689392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,16,32,0,0.0192208006978035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,16,64,0,0.01884479969739914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,32,1,0,0.027750399708747864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,2,32768,32,0,14.387037658691407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,32,2,0,0.01624799966812134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,32,4,0,0.017392000555992125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,32,16,0,0.016735999286174773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,32,32,0,0.015673600137233734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,32,64,0,0.015211200714111328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,32,1,0,0.04009920060634613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,32,2,0,0.02314720004796982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,32,4,0,0.022700800001621245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,2,32768,64,0,7.364475250244141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,32,16,0,0.023839999735355378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,32,32,0,0.022547200322151184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,32,64,0,0.022291199862957002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,64,1,0,0.053604799509048465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,64,2,0,0.026713600754737853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,64,4,0,0.017342400550842286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,64,8,0,0.016407999396324157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,64,16,0,0.016271999478340148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,64,32,0,0.015137599408626556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,64,64,0,0.014422400295734406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,64,1,0,0.05950080156326294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,64,2,0,0.03551680147647858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,64,4,0,0.023529599606990814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,64,8,0,0.022593599557876588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,64,16,0,0.022316800057888032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,32,8,0,0.015347200632095336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,64,64,0,0.020308800041675568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,128,1,0,0.08814079761505127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,128,2,0,0.05170239806175232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,128,4,0,0.029311999678611755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,128,8,0,0.018228800594806673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,32,8,0,0.022115199267864226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,128,16,0,0.01780479997396469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,128,32,0,0.01740639954805374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,128,64,0,0.016804799437522888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,128,1,0,0.10054240226745606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,128,2,0,0.06220160126686096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,128,4,0,0.03910239934921265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,128,8,0,0.025014400482177734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,128,16,0,0.0245728000998497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,128,32,0,0.024145600199699403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,128,64,0,0.023104000091552734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,256,1,0,0.18158719539642335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,256,2,0,0.10066560506820679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,256,4,0,0.058494400978088376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,256,8,0,0.035128000378608706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,256,16,0,0.022654399275779724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,256,32,0,0.022217600047588347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,64,32,0,0.02202720046043396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,256,64,0,0.022168000042438508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,256,2,0,0.11198240518569946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,256,4,0,0.06986079812049865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,256,8,0,0.04650239944458008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,256,16,0,0.02988319993019104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,256,32,0,0.02951200008392334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,256,64,0,0.029422399401664735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,512,1,0,0.4423696041107178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,512,2,0,0.2347104072570801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,512,4,0,0.12767200469970702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,512,8,0,0.07162719964981079
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,512,16,0,0.05555199980735779
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,512,32,0,0.036148801445961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,512,64,0,0.04020160138607025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,512,1,0,0.4532000064849854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,256,1,0,0.19194719791412354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,512,4,0,0.14127520322799683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,512,2,0,0.2473088026046753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,512,8,0,0.08526080250740051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,512,16,0,0.06939679980278016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,512,32,0,0.04864799976348877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,512,64,0,0.053390401601791385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,1024,4,0,0.34333600997924807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,1024,2,0,0.6507152080535888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,1024,8,0,0.1805791974067688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,1024,16,0,0.10145280361175538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,1024,32,0,0.0798255980014801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,1024,64,0,0.055169600248336795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,1024,1,0,1.2613455772399902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,1024,4,0,0.35774080753326415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,1024,2,0,0.6696112155914307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,1024,8,0,0.1959504008293152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,1024,16,0,0.11806399822235107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,1024,32,0,0.09683679938316345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,1024,64,0,0.07013440132141113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,1536,2,0,1.2994352340698243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,1536,4,0,0.6884607791900634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,1536,1,0,2.550886344909668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,1536,8,0,0.3536992073059082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,1536,16,0,0.20278561115264893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,1536,32,0,0.1273136019706726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,1536,64,0,0.09433760046958924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,1024,1,0,1.273691177368164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,1536,4,0,0.6754079818725586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,1536,2,0,1.3379599571228027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,1536,8,0,0.3569648027420044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,1536,16,0,0.20760641098022461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,1536,32,0,0.14408799409866332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,1536,64,0,0.11293760538101197
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,1536,1,0,2.5098352432250977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,2048,4,0,1.123852825164795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,2048,2,0,2.178206443786621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,2048,8,0,0.5884496212005615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,2048,32,0,0.1728384017944336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,2048,16,0,0.31699519157409667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,2048,64,0,0.1381999969482422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,2048,1,0,4.306507110595703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,2048,2,0,2.1781232833862303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,2048,4,0,1.1696816444396974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,2048,16,0,0.32999041080474856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,2048,8,0,0.6088399887084961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,2048,64,0,0.16055519580841066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,2048,32,0,0.19351199865341187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,2048,1,0,4.249299240112305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,3072,4,0,2.255161666870117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,3072,8,0,1.2175632476806642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,3072,2,0,4.699800109863281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,3072,16,0,0.6328767776489258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,3072,64,0,0.22957279682159423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,3072,32,0,0.3269184112548828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,3072,1,0,9.378892517089843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,3072,4,0,2.2870655059814453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,3072,8,0,1.1969008445739746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,3072,2,0,4.395355224609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,3072,16,0,0.6364272117614747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,3072,32,0,0.3696000099182129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,3072,64,0,0.25340321063995364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,3072,1,0,8.976630401611327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,4096,4,0,4.0603790283203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,2,32768,4,0,113.6762939453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,4096,8,0,2.1466432571411134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,4096,16,0,1.0121808052062988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,4096,64,0,0.29826400279998777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,4096,32,0,0.5713215827941894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,4096,2,0,8.447889709472657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,4096,1,0,15.815097045898437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,4096,2,0,8.123162841796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,4096,8,0,1.9997568130493164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,4096,16,0,1.017512035369873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,4096,4,0,3.8873695373535155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,4096,32,0,0.5461440086364746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,4096,64,0,0.3357935905456543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,4096,1,0,15.683934020996094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,6144,8,0,4.355380630493164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,6144,4,0,9.020073699951173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,6144,16,0,2.376300811767578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,6144,32,0,1.1375712394714355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,6144,64,0,0.6622064113616943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,6144,2,0,17.476344299316406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,6144,4,0,8.386688232421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,6144,2,0,17.167701721191406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,6144,8,0,4.235240173339844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,6144,16,0,2.1211423873901367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,6144,64,0,0.6585855960845948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,6144,32,0,1.2119456291198731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,2,32768,2,0,235.5247314453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,6144,1,0,36.149444580078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,8192,8,0,7.675865936279297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,6144,1,0,34.727334594726564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,8192,32,0,1.8966447830200195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,8192,16,0,4.278079986572266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,8192,64,0,1.0253536224365234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,8192,4,0,15.52236328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,8192,2,0,30.955438232421876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,8192,4,0,14.798379516601562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,8192,16,0,3.735015869140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,8192,8,0,7.737142181396484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,8192,32,0,1.965822410583496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,8192,64,0,1.1826911926269532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,8192,2,0,30.353521728515624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,2,32768,2,0,226.126953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,8192,1,0,61.72515869140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,10240,8,0,12.101900482177735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,10240,32,0,3.014468765258789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,10240,16,0,5.98687858581543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,10240,64,0,1.518396759033203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,10240,4,0,24.113917541503906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,8192,1,0,59.53043823242187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,10240,8,0,11.694385528564453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,10240,2,0,48.812164306640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,10240,32,0,2.9684240341186525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,10240,16,0,6.212812805175782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,10240,4,0,23.270295715332033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,10240,64,0,1.5707327842712402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,10240,2,0,45.961813354492186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,12288,4,0,35.227935791015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,12288,8,0,17.391192626953124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,10240,1,0,94.88089599609376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,12288,32,0,4.347608184814453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,12288,64,0,2.143027114868164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,12288,16,0,8.827744293212891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,10240,1,0,91.38048706054687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,12288,2,0,69.42283325195312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,12288,16,0,8.231839752197265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,12288,4,0,33.93023681640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,12288,8,0,16.52031555175781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,12288,64,0,2.1623247146606444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,12288,32,0,4.28130874633789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,12288,2,0,65.22142333984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,12288,1,0,137.5716552734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,4,16384,8,0,30.38551025390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,4,16384,4,0,61.16013793945312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,4,16384,32,0,7.542366027832031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,2,32768,1,0,476.24560546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,4,16384,64,0,3.9360225677490233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,4,16384,16,0,15.444964599609374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,12288,1,0,130.83944091796874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,2,32768,1,0,455.484033203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,4,16384,8,0,29.249569702148438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,4,16384,64,0,3.884316635131836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,1,1,0,0.01542080044746399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,1,2,0,0.014961600303649902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,1,4,0,0.014510400593280792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,1,8,0,0.013249599933624267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,1,16,0,0.013172799348831176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,1,32,0,0.013196800649166108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,1,64,0,0.013120000064373017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,1,1,0,0.020876799523830415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,4,16384,32,0,7.4168754577636715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,1,2,0,0.01921759992837906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,1,8,0,0.018654400110244752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,1,4,0,0.020020799338817598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,4,16384,16,0,14.750700378417969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,1,32,0,0.01960960030555725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,16,1,0,0.030132800340652466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,16,2,0,0.017407999932765962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,16,4,0,0.016820800304412842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,16,8,0,0.01610559970140457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,16,16,0,0.014827199280261993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,16,32,0,0.014246399700641631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,16,64,0,0.013756799697875976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,16,1,0,0.03559359908103943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,16,2,0,0.023236800730228425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,16,4,0,0.022836799919605254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,16,8,0,0.02210240066051483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,1,16,0,0.01867839992046356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,16,16,0,0.020127999782562255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,16,32,0,0.019732800126075745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,1,64,0,0.020343999564647674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,16,64,0,0.01921440064907074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,32,1,0,0.04880799949169159
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,32,2,0,0.029150399565696716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,32,4,0,0.016388800740242005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,32,16,0,0.015491199493408204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,32,32,0,0.014193600416183472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,32,64,0,0.013920000195503235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,32,1,0,0.05833920240402222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,32,2,0,0.034004798531532286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,32,4,0,0.02287999987602234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,32,8,0,0.022075200080871583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,32,16,0,0.022137600183486938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,32,32,0,0.020660799741744996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,32,64,0,0.02024800032377243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,64,1,0,0.08458880186080933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,64,2,0,0.04927679896354675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,64,4,0,0.02671839892864227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,64,8,0,0.017283199727535246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,64,16,0,0.01653600037097931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,64,32,0,0.016414399445056915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,64,64,0,0.015459200739860535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,64,1,0,0.09454560279846191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,64,2,0,0.059217602014541626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,64,4,0,0.03524639904499054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,64,8,0,0.023827199637889863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,64,16,0,0.02317280024290085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,64,32,0,0.02280319929122925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,32,8,0,0.01594720035791397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,64,64,0,0.02165600061416626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,128,2,0,0.0889248013496399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,128,1,0,0.16047999858856202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,128,4,0,0.05301759839057922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,128,8,0,0.031009599566459656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,128,16,0,0.019705599546432494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,128,32,0,0.019472000002861024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,128,64,0,0.019326399266719817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,128,1,0,0.1694383978843689
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,128,2,0,0.09991040229797363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,128,4,0,0.06320639848709106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,128,8,0,0.041433599591255185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,128,16,0,0.027011200785636902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,128,32,0,0.02614719867706299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,128,64,0,0.02640959918498993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,256,2,0,0.18668960332870482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,256,1,0,0.3430335998535156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,256,8,0,0.06097440123558044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,256,16,0,0.03946399986743927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,256,32,0,0.025798401236534117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,256,64,0,0.02558239996433258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,256,1,0,0.3554255962371826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,256,4,0,0.11630400419235229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,256,2,0,0.19983839988708496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,256,8,0,0.07384480237960815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,256,16,0,0.05367839932441711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,256,32,0,0.03613759875297547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,256,64,0,0.03523359894752502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,512,2,0,0.4518271923065186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,512,1,0,0.8581168174743652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,512,4,0,0.2425407886505127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,512,8,0,0.13077919483184813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,512,16,0,0.09561920166015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,512,32,0,0.06506239771842956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,256,4,0,0.10401279926300049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,512,2,0,0.4699728012084961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,512,1,0,0.8726736068725586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,512,4,0,0.26118080615997313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,512,8,0,0.14689919948577881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,512,16,0,0.11244159936904907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,512,32,0,0.08136320114135742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,4,16384,2,0,122.4527099609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,512,64,0,0.058278399705886844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,1024,4,0,0.6735360145568847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,1024,8,0,0.34772319793701173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,1024,16,0,0.19226560592651368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,1024,2,0,1.50764799118042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,1024,32,0,0.13927040100097657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,1024,64,0,0.09945120215415955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,1024,1,0,2.5522415161132814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,4,16384,4,0,58.628955078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,1024,4,0,0.7362304210662842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,1024,2,0,1.3050576210021974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,1024,8,0,0.4140431880950928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,1024,16,0,0.22525599002838134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,1024,32,0,0.1616431951522827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,1024,64,0,0.12602560520172118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,1024,1,0,2.519726371765137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,512,64,0,0.04289279878139496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,1536,8,0,0.6817344188690185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,1536,4,0,1.3156111717224122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,1536,16,0,0.3520591974258423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,1536,32,0,0.23435521125793457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,1536,64,0,0.15696640014648439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,1536,2,0,2.587420845031738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,1536,4,0,1.3215888023376465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,1536,8,0,0.7527599811553956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,1536,16,0,0.4017615795135498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,1536,32,0,0.27425599098205566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,1536,64,0,0.19586559534072875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,1536,2,0,2.6035087585449217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,1536,1,0,5.118313598632812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,1536,1,0,5.12138557434082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,2048,4,0,2.2682559967041014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,2048,8,0,1.077340793609619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,2048,16,0,0.576367998123169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,2048,64,0,0.22693440914154053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,2048,2,0,4.523172760009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,2048,32,0,0.33143680095672606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,2048,1,0,8.958248138427734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,2048,8,0,1.106332778930664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,2048,32,0,0.35727519989013673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,2048,64,0,0.26640639305114744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,2048,4,0,2.1590496063232423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,2048,1,0,8.538299560546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,2048,2,0,4.361439895629883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,2048,16,0,0.6567408084869385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,3072,16,0,1.1505680084228516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,3072,8,0,2.2574335098266602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,3072,32,0,0.6311183929443359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,3072,64,0,0.4323999881744385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,3072,4,0,4.6165824890136715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,3072,2,0,9.358977508544921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,3072,8,0,2.288830375671387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,3072,4,0,4.6804351806640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,3072,32,0,0.7351776123046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,3072,16,0,1.22117280960083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,3072,64,0,0.49169440269470216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,3072,2,0,9.162433624267578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,3072,1,0,18.361830139160155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,4096,8,0,3.9948734283447265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,4096,4,0,7.996009826660156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,3072,1,0,18.026229858398438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,4096,64,0,0.5904975891113281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,4096,32,0,1.1066896438598632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,4096,16,0,2.0475088119506837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,4096,2,0,16.485052490234374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,4096,8,0,3.9056095123291015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,4096,4,0,7.911255645751953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,4096,16,0,2.032980728149414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,4096,32,0,1.1469200134277344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,4096,64,0,0.6381616115570068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,4,16384,2,0,115.89832763671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,4096,2,0,16.841795349121092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,4096,1,0,33.61650390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,6144,8,0,9.058838653564454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,6144,16,0,4.382353591918945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,6144,64,0,1.2228912353515624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,6144,32,0,2.2107152938842773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,4096,1,0,32.633786010742185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,6144,4,0,18.203816223144532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,6144,8,0,8.584519958496093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,6144,16,0,4.348563385009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,6144,2,0,35.877587890625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,6144,32,0,2.2913135528564452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,6144,4,0,17.64653625488281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,6144,64,0,1.2861231803894042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,6144,2,0,35.02037658691406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,4,16384,1,0,240.134228515625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,8,8192,8,0,15.33773193359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,8,8192,16,0,7.582278442382813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,8,8192,4,0,31.413067626953126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,8,8192,64,0,2.0383567810058594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,8,8192,32,0,3.9089279174804688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,6144,1,0,71.8973388671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,6144,1,0,69.14630126953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,8,8192,8,0,14.865768432617188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,8,8192,32,0,3.820761489868164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,8,8192,16,0,7.496086120605469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,1,1,0,0.015454399585723876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,1,2,0,0.014672000706195832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,8,8192,64,0,2.087838363647461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,1,4,0,0.013575999438762665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,1,16,0,0.013147200644016265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,1,32,0,0.012761600315570831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,1,64,0,0.012670400738716125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,1,1,0,0.02083359956741333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,1,2,0,0.01923519968986511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,1,4,0,0.018571199476718904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,1,8,0,0.018110400438308714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,1,16,0,0.018139199912548067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,1,32,0,0.018222400546073915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,1,64,0,0.01807519942522049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,16,1,0,0.04884639978408813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,16,2,0,0.025446400046348572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,16,4,0,0.01642400026321411
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,16,8,0,0.015747199952602386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,16,16,0,0.015455999970436096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,16,32,0,0.014044800400733947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,16,64,0,0.013809600472450256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,16,1,0,0.05920640230178833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,16,2,0,0.03510879874229431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,16,4,0,0.023156799376010895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,16,8,0,0.021964800357818604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,16,16,0,0.022235199809074402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,16,32,0,0.02141920030117035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,16,64,0,0.020078399777412416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,32,1,0,0.08428320288658142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,1,8,0,0.013302400708198547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,32,2,0,0.049639999866485596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,32,4,0,0.027030399441719054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,32,8,0,0.017185600101947786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,32,16,0,0.01679359972476959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,32,32,0,0.016327999532222748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,32,64,0,0.015022400021553039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,32,1,0,0.09475359916687012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,32,2,0,0.05976799726486206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,32,4,0,0.0361871987581253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,32,8,0,0.023887999355793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,32,16,0,0.02337439954280853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,32,32,0,0.02258239984512329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,32,64,0,0.02152640074491501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,64,1,0,0.1534160017967224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,64,2,0,0.08630880117416381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,64,4,0,0.051583999395370485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,64,8,0,0.029046401381492615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,8,8192,2,0,63.2653564453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,64,16,0,0.018812799453735353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,64,64,0,0.018268799781799315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,64,32,0,0.019918400049209594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,64,1,0,0.16139520406723024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,8,8192,4,0,30.3584228515625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,64,4,0,0.06114879846572876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,64,2,0,0.10074080228805542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,64,32,0,0.02614080011844635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,64,64,0,0.025999999046325682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,128,1,0,0.300927996635437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,128,4,0,0.09063680171966552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,128,8,0,0.05623840093612671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,128,16,0,0.03460319936275482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,128,32,0,0.022862400114536285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,128,64,0,0.022595199942588805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,128,1,0,0.31255040168762205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,128,2,0,0.17580959796905518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,128,4,0,0.10337920188903808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,128,8,0,0.06913599967956544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,128,16,0,0.04865919947624207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,128,32,0,0.03235200047492981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,128,64,0,0.03186880052089691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,64,16,0,0.025891199707984924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,256,2,0,0.35570878982543946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,256,1,0,0.6696832180023193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,256,4,0,0.1948047995567322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,256,8,0,0.10666879415512084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,256,16,0,0.068094402551651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,256,32,0,0.047312000393867494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,256,64,0,0.03204320073127746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,256,2,0,0.3717328071594238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,256,1,0,0.6853424072265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,256,4,0,0.21093919277191162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,256,16,0,0.08450880050659179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,256,32,0,0.06359999775886535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,256,64,0,0.0469215989112854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,64,8,0,0.04538399875164032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,128,2,0,0.1620959997177124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,512,1,0,1.69378719329834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,512,2,0,0.8906399726867675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,512,8,0,0.2615072011947632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,512,16,0,0.17274240255355836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,512,4,0,0.47707839012145997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,512,64,0,0.07878559827804565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,512,32,0,0.1130560040473938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,256,8,0,0.12301280498504638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,512,4,0,0.5141695976257324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,512,8,0,0.27854559421539304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,512,16,0,0.20028960704803467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,512,32,0,0.13591840267181396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,4,16384,1,0,229.0519775390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,512,1,0,1.7251760482788085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,1024,4,0,1.3338111877441405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,1024,8,0,0.6863632202148438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,512,2,0,0.9177951812744141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,1024,16,0,0.3659600019454956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,512,64,0,0.1052240014076233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,1024,32,0,0.25195679664611814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,1024,64,0,0.1675376057624817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,1024,4,0,1.3554143905639648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,1024,1,0,5.406472015380859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,1024,16,0,0.4449039936065674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,1024,8,0,0.7281136035919189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,1024,2,0,2.601443290710449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,1024,64,0,0.2101439952850342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,1024,32,0,0.30444960594177245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,1024,2,0,2.6372976303100586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,1536,8,0,1.3193008422851562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,1024,1,0,5.144417572021484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,1536,16,0,0.7058784008026123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,1536,4,0,2.7248687744140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,1536,32,0,0.4371232032775879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,1536,64,0,0.28063039779663085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,1536,2,0,5.449859237670898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,1536,4,0,2.657329559326172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,1536,16,0,0.7751488208770752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,1536,8,0,1.390721607208252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,1536,64,0,0.3473615884780884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,1536,2,0,5.304729461669922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,8,8192,2,0,59.787322998046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,1536,1,0,10.557676696777344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,1536,32,0,0.503872013092041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,1536,1,0,10.309349060058594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,2048,8,0,2.1534751892089843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,2048,32,0,0.6303264141082764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,2048,16,0,1.1545568466186524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,2048,4,0,4.74200325012207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,2048,64,0,0.43093118667602537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,2048,2,0,8.973538970947265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,2048,4,0,4.4382366180419925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,2048,8,0,2.2345455169677733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,2048,32,0,0.6906479835510254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,2048,16,0,1.168889617919922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,2048,64,0,0.4873631954193115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,2048,2,0,8.944900512695312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,2048,1,0,18.003878784179687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,3072,16,0,2.606982421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,3072,8,0,4.803569412231445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,3072,32,0,1.3811344146728515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,3072,64,0,0.8164912223815918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,3072,4,0,9.539868927001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,2048,1,0,18.51097869873047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,3072,8,0,4.871067047119141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,3072,2,0,18.59708251953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,8,8192,1,0,123.985986328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,3072,4,0,9.122433471679688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,3072,16,0,2.45654239654541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,3072,32,0,1.304753589630127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,3072,64,0,0.9220399856567383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,3072,2,0,18.65717468261719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,16,4096,8,0,8.359568023681641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,16,4096,32,0,2.1910816192626954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,16,4096,16,0,4.238070297241211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,16,4096,64,0,1.1865551948547364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,3072,1,0,38.89661865234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,16,4096,4,0,17.143037414550783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,8,8192,1,0,118.12142333984374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,3072,1,0,36.942874145507815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,16,4096,8,0,7.773662567138672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,16,4096,64,0,1.2655344009399414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,32,1,1,0,0.015619200468063355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,1,2,0,0.01472959965467453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,1,4,0,0.013804799318313599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,1,8,0,0.013249599933624267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,16,4096,32,0,2.3099199295043946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,1,32,0,0.01462559998035431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,1,64,0,0.014654399454593658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,32,1,1,0,0.023227199912071228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,1,2,0,0.02215680032968521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,1,4,0,0.021164800226688384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,1,8,0,0.0196943998336792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,16,4096,16,0,4.143703842163086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,1,16,0,0.01982080042362213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,1,64,0,0.01966720074415207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,1,32,0,0.020603199303150178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,32,16,1,0,0.08740000128746032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,16,2,0,0.05243840217590332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,16,4,0,0.02760159969329834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,16,8,0,0.017980800569057466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,16,16,0,0.017209599912166595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,16,32,0,0.016571199893951415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,16,64,0,0.01526239961385727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,32,16,1,0,0.09721760153770446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,16,4,0,0.03746559917926788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,16,8,0,0.02480800002813339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,16,16,0,0.023665599524974823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,16,32,0,0.02322079986333847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,16,64,0,0.022254399955272675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,32,32,1,0,0.15029759407043458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,32,2,0,0.08662400245666504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,1,16,0,0.013089600205421447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,32,8,0,0.0324400007724762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,32,16,0,0.019449600577354433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,32,32,0,0.018681600689888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,32,64,0,0.018240000307559966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,32,32,1,0,0.1637776017189026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,16,4096,2,0,33.94309997558594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,32,2,0,0.09814559817314147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,32,8,0,0.04177600145339966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,32,4,0,0.06979039907455445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,32,16,0,0.026790401339530943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,32,64,0,0.02542079985141754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,32,32,0,0.027988800406455995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,32,64,1,0,0.2879120111465454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,64,2,0,0.16429280042648314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,64,8,0,0.05846400260925293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,64,16,0,0.0353983998298645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,64,32,0,0.02402079999446869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,64,64,0,0.02342880070209503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,32,64,1,0,0.3032847881317139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,32,4,0,0.051179200410842896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,64,4,0,0.10285919904708862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,64,8,0,0.0675711989402771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,64,16,0,0.04812960028648376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,64,32,0,0.03325600028038025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,64,64,0,0.031439998745918275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,32,128,1,0,0.5869167804718017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,128,2,0,0.31066720485687255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,16,4096,4,0,16.5673583984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,64,4,0,0.09025760293006897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,128,16,0,0.06343520283699036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,128,8,0,0.10237760543823242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,128,32,0,0.0440416008234024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,128,64,0,0.029625600576400755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,64,2,0,0.1670367956161499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,32,128,1,0,0.6021215915679932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,128,2,0,0.3260623931884766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,128,4,0,0.18157919645309448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,128,16,0,0.07988799810409546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,128,32,0,0.05979520082473755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,128,64,0,0.04441120028495789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,16,2,0,0.06079679727554321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,128,4,0,0.16587040424346924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,32,256,1,0,1.316819190979004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,256,4,0,0.37709760665893555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,256,2,0,0.704857587814331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,256,16,0,0.11972160339355468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,256,8,0,0.19999200105667114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,256,32,0,0.08209120035171509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,256,64,0,0.06060640215873718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,256,4,0,0.40159358978271487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,256,2,0,0.7225679874420166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,256,8,0,0.22166879177093507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,128,8,0,0.11477119922637939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,256,16,0,0.1444111943244934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,32,256,1,0,1.358339214324951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,512,2,0,1.7828447341918945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,512,4,0,0.9534015655517578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,16,4096,2,0,32.03786926269531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,512,8,0,0.49443039894104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,512,16,0,0.33218719959259035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,512,32,0,0.20159521102905273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,512,64,0,0.13917280435562135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,512,4,0,0.9936127662658691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,512,2,0,1.7968767166137696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,512,8,0,0.5373248100280762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,512,32,0,0.2458751916885376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,512,16,0,0.36802239418029786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,32,512,1,0,3.404596710205078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,512,64,0,0.18253439664840698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,32,512,1,0,3.4441089630126953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,1024,8,0,1.3776080131530761
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,1024,4,0,2.709987258911133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,1024,16,0,0.7470320224761963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,256,32,0,0.10545120239257813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,1024,32,0,0.5027391910552979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,1024,64,0,0.31694080829620364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,1024,2,0,5.349591827392578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,1024,4,0,2.762686347961426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,1024,8,0,1.438657569885254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,256,64,0,0.08522400259971619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,1024,16,0,0.8086607933044434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,1024,32,0,0.5515120029449463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,1024,2,0,5.327328109741211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,32,1024,1,0,10.620220947265626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,1024,64,0,0.38830718994140623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,16,4096,1,0,66.14721069335937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,1536,8,0,2.69463996887207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,1536,16,0,1.3764431953430176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,32,1024,1,0,10.263620758056641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,1536,64,0,0.5263296127319336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,1536,32,0,0.9385312080383301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,1536,4,0,5.348748779296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,1536,16,0,1.4855584144592284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,1536,8,0,2.7016368865966798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,1536,32,0,0.9559344291687012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,1536,64,0,0.6505263805389404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,1536,2,0,10.853096008300781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,1536,4,0,5.264134216308594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,16,4096,1,0,62.83221435546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,32,2048,8,0,4.509043121337891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,1536,2,0,10.405323028564453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,32,2048,16,0,2.2661951065063475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,32,2048,32,0,1.259836769104004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,32,2048,64,0,0.7848959922790527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,32,2048,4,0,9.138823699951171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,32,1536,1,0,21.192424011230468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,32,2048,16,0,2.594950485229492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,32,1536,1,0,20.45809326171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,32,2048,8,0,4.902756881713867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,32,2048,32,0,1.4486895561218263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,1,2,0,0.016944000124931337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,1,4,0,0.015572799742221833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,1,8,0,0.01526080071926117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,1,16,0,0.014214399456977844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,32,2048,4,0,8.914126586914062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,1,32,0,0.014528000354766845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,32,2048,64,0,0.9741328239440918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,64,1,1,0,0.02396160066127777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,1,64,0,0.014584000408649444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,1,4,0,0.021028800308704375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,1,8,0,0.020351999998092653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,1,32,0,0.019912000000476836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,1,64,0,0.019939200580120088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,32,2048,2,0,18.293055725097656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,64,16,1,0,0.15228320360183717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,16,4,0,0.052964800596237184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,16,8,0,0.030107200145721436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,16,16,0,0.02018879950046539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,16,32,0,0.018985599279403687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,16,64,0,0.017441600561141968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,64,1,1,0,0.019463999569416045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,64,16,1,0,0.16403520107269287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,16,2,0,0.09853600263595581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,16,4,0,0.0633135974407196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,16,8,0,0.041407999396324155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,16,16,0,0.02730399966239929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,1,2,0,0.021014399826526642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,1,16,0,0.02059520035982132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,16,64,0,0.024556800723075867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,32,2,0,0.158624005317688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,64,32,1,0,0.288588809967041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,32,4,0,0.09081599712371827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,32,8,0,0.05637279748916626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,32,16,0,0.03452799916267395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,32,32,0,0.023203200101852416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,32,64,0,0.02197919934988022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,16,2,0,0.10065599679946899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,32,2,0,0.16742080450057983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,64,32,1,0,0.3010416030883789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,32,4,0,0.1038640022277832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,32,8,0,0.06839519739151001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,32,16,0,0.04879519939422607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,32,32,0,0.03307519853115082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,32,64,0,0.03149920105934143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,64,4,0,0.16216800212860108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,64,2,0,0.2978559970855713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,64,8,0,0.0975279986858368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,64,16,0,0.06219999790191651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,64,32,0,0.04261119961738587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,64,64,0,0.029679998755455017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,64,2,0,0.30864319801330564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,64,64,1,0,0.5805840015411377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,64,4,0,0.17837920188903808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,64,16,0,0.07865279912948608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,16,32,0,0.025856000185012818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,64,32,0,0.05878880023956299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,64,64,0,0.04480159878730774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,32,2048,2,0,18.341769409179687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,64,128,1,0,1.1562911987304687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,128,4,0,0.3410367965698242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,128,8,0,0.18386399745941162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,128,16,0,0.11633119583129883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,128,64,0,0.056815999746322635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,64,64,1,0,0.591929578781128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,128,2,0,0.6336287975311279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,64,8,0,0.11340479850769043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,64,128,1,0,1.1803999900817872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,128,4,0,0.3413055896759033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,128,8,0,0.20448479652404786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,128,64,0,0.0813535988330841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,128,16,0,0.13628159761428832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,128,2,0,0.6084671974182129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,256,4,0,0.7408944129943847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,256,8,0,0.38248319625854493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,256,16,0,0.22477281093597412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,256,32,0,0.14882559776306153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,256,2,0,1.378121566772461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,256,64,0,0.11003999710083008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,256,4,0,0.7834224224090576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,64,256,1,0,2.622233581542969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,256,8,0,0.4228672027587891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,256,2,0,1.4201904296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,256,16,0,0.2785072088241577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,128,32,0,0.10127840042114258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,256,32,0,0.19010560512542723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,256,64,0,0.15338879823684692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,64,256,1,0,2.6573007583618162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,512,8,0,0.9744432449340821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,128,32,0,0.07731680274009704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,512,16,0,0.6260511875152588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,512,32,0,0.3850831985473633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,512,4,0,1.8570880889892578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,32,2048,1,0,36.577423095703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,512,2,0,3.5466495513916017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,512,4,0,1.9847248077392579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,512,8,0,1.116153621673584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,512,16,0,0.7118720054626465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,512,64,0,0.34288480281829836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,512,64,0,0.25601279735565186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,512,2,0,3.611606216430664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,64,512,1,0,7.008062744140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,64,1024,16,0,1.413263988494873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,32,2048,1,0,34.95809631347656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,64,1024,8,0,2.7432735443115233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,512,32,0,0.46657438278198243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,64,512,1,0,6.881905364990234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,64,1024,32,0,0.9163215637207032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,64,1024,64,0,0.5988656044006347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,64,1024,4,0,5.495195388793945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,64,1024,16,0,1.5859984397888183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,64,1024,32,0,1.0705439567565918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,64,1024,64,0,0.7745359897613525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,64,1024,8,0,2.8463855743408204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,128,1,1,0,0.03094559907913208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,1,2,0,0.01801919937133789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,1,4,0,0.016169600188732147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,1,8,0,0.015585599839687348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,1,16,0,0.0160303995013237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,1,32,0,0.015600000321865082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,1,64,0,0.01528480052947998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,128,1,1,0,0.03779520094394684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,1,4,0,0.02290399968624115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,1,8,0,0.021406400203704833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,1,16,0,0.021264000236988066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,1,32,0,0.021169599890708924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,1,64,0,0.02123039960861206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,64,1024,4,0,5.463409423828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,128,16,1,0,0.29027519226074217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,16,2,0,0.16425440311431885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,16,4,0,0.09148799777030944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,16,16,0,0.03570719957351685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,16,8,0,0.05841919779777527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,16,32,0,0.0234607994556427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,16,64,0,0.022103999555110932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,64,1024,2,0,10.93353271484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,128,16,1,0,0.30165920257568357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,16,2,0,0.1680240035057068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,16,4,0,0.11533600091934204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,16,16,0,0.048732799291610715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,16,64,0,0.03170399963855743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,16,32,0,0.03557760119438171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,32,2,0,0.30335679054260256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,128,32,1,0,0.5712175846099854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,32,4,0,0.16746400594711303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,32,8,0,0.09839360117912292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,32,16,0,0.062745600938797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,32,32,0,0.042985600233078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,128,32,1,0,0.5816319942474365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,32,2,0,0.30852320194244387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,32,4,0,0.17845280170440675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,64,1024,2,0,10.474540710449219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,32,8,0,0.11467360258102417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,32,32,0,0.05975840091705322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,32,16,0,0.0823472023010254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,32,64,0,0.044563201069831845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,1,2,0,0.02346239984035492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,64,2,0,0.5780543804168701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,64,4,0,0.31818881034851076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,64,8,0,0.17730560302734374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,32,64,0,0.03012799918651581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,64,32,0,0.07497280240058898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,128,64,1,0,1.1241951942443849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,64,64,0,0.055190402269363406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,64,8,0,0.20043680667877198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,64,4,0,0.3329760074615479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,64,2,0,0.6034687995910645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,64,32,0,0.09942079782485962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,64,64,0,0.08012160062789916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,128,64,1,0,1.1492048263549806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,16,8,0,0.06913440227508545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,128,4,0,0.6188000202178955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,128,2,0,1.2020848274230957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,128,8,0,0.3464287996292114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,128,64,0,0.10589920282363892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,128,32,0,0.1442863941192627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,128,128,1,0,2.275579261779785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,64,16,0,0.11190880537033081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,64,1024,1,0,21.14575653076172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,128,8,0,0.39059998989105227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,128,2,0,1.243892765045166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,64,16,0,0.13609440326690675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,128,4,0,0.6569632053375244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,128,32,0,0.1843775987625122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,128,16,0,0.2756592035293579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,128,64,0,0.14867039918899536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,256,16,0,0.4338031768798828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,128,128,1,0,2.3376256942749025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,256,32,0,0.27935841083526614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,256,8,0,0.7658751964569092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,256,64,0,0.20962240695953369
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,128,16,0,0.20884160995483397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,256,4,0,1.478212833404541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,256,16,0,0.5153232097625733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,256,8,0,0.8278448104858398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,256,2,0,2.7222320556640627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,256,32,0,0.3603424072265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,64,1024,1,0,21.015704345703124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,256,4,0,1.5520959854125977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,256,64,0,0.2835103988647461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,256,2,0,2.8371984481811525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,128,512,16,0,1.2361087799072266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,128,512,8,0,1.925881576538086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,128,512,64,0,0.4891615867614746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,128,256,1,0,5.310691070556641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,128,512,4,0,4.137408065795898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,128,256,1,0,5.233369445800781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,128,512,32,0,0.7420959949493409
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,128,512,2,0,7.112283325195312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,128,512,8,0,2.193129539489746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,128,512,32,0,0.8932944297790527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,256,1,1,0,0.055796802043914795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,1,4,0,0.02309119999408722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,128,512,4,0,3.8562286376953123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,1,2,0,0.031240001320838928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,1,8,0,0.022767999768257143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,128,512,64,0,0.6501887798309326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,1,64,0,0.0222463995218277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,1,16,0,0.022017599642276765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,1,2,0,0.03452799916267395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,256,1,1,0,0.0622160017490387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,1,8,0,0.02831839919090271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,1,16,0,0.028321599960327147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,1,32,0,0.028240001201629637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,1,64,0,0.028182399272918702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,128,512,2,0,7.146614074707031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,256,16,1,0,0.5648848056793213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,16,4,0,0.16454399824142457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,16,2,0,0.31281280517578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,16,8,0,0.10014560222625732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,16,16,0,0.06589919924736024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,16,32,0,0.04204800128936768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,16,64,0,0.035231998562812804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,128,512,16,0,1.3971664428710937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,16,2,0,0.3092959880828857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,256,16,1,0,0.5778255939483643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,16,4,0,0.1800927996635437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,16,8,0,0.1149440050125122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,16,32,0,0.05971999764442444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,16,16,0,0.08047999739646912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,1,4,0,0.029524800181388856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,32,4,0,0.3116688013076782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,32,2,0,0.5741151809692383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,32,16,0,0.11287200450897217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,256,32,1,0,1.167191982269287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,32,32,0,0.07599679827690124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,128,512,1,0,14.118788146972657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,32,64,0,0.0562720000743866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,32,4,0,0.3339200019836426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,32,2,0,0.6483839988708496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,256,32,1,0,1.1478544235229493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,1,32,0,0.022350400686264038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,32,64,0,0.08220000267028808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,16,64,0,0.049491199851036075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,128,512,1,0,13.697615051269532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,64,2,0,1.1406959533691405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,64,4,0,0.6078000068664551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,32,8,0,0.1788607954978943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,64,64,0,0.10467519760131835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,64,32,0,0.14285600185394287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,256,64,1,0,2.2369232177734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,32,8,0,0.2031696081161499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,64,2,0,1.180947208404541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,64,4,0,0.6528751850128174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,32,32,0,0.10496959686279297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,64,8,0,0.38236958980560304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,64,16,0,0.2507152080535889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,64,32,0,0.18510880470275878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,256,64,1,0,2.2565711975097655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,64,8,0,0.3621936082839966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,128,8,0,0.6718016147613526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,128,4,0,1.2216464042663575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,128,16,0,0.40054879188537595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,128,32,0,0.26330080032348635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,128,64,0,0.19746559858322144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,128,2,0,2.387076759338379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,32,16,0,0.13748159408569335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,128,16,0,0.48360161781311034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,64,64,0,0.14767999649047853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,128,8,0,0.751470422744751
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,128,4,0,1.3061264038085938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,128,32,0,0.3460016012191772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,128,64,0,0.2772639989852905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,256,128,1,0,4.566241455078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,64,16,0,0.20888800621032716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,128,2,0,2.456328010559082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,4,256,256,32,0,0.5402448177337646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,2,256,256,64,0,0.3856368064880371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,8,256,256,16,0,0.8699808120727539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,256,128,1,0,4.638193511962891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,32,256,256,4,0,2.942851257324219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,8,256,256,16,0,1.0056639671325684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,32,256,256,4,0,3.0738128662109374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,16,256,256,8,0,1.4801504135131835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,16,256,256,8,0,1.6205007553100585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,4,256,256,32,0,0.6941023826599121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,2,256,256,64,0,0.5389872074127198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,64,256,256,2,0,5.579291152954101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,64,256,256,2,0,5.460675048828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,bfloat16,128,256,256,1,0,10.389473724365235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,mla_context,flash_attention,bfloat16,fp8,128,256,256,1,0,10.545295715332031
