framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,1,0.12811360359191895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,1,0.15580320358276367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,3,0.12814719676971437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,3,0.15569280385971068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,7,0.12838399410247803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,7,0.15578399896621703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,15,0.12994400262832642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,15,0.15587999820709228
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,31,0.1349503993988037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,31,0.155457603931427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,63,0.13840960264205932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,63,0.16615840196609497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,127,0.14335039854049683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,127,0.17477600574493407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,255,0.21006240844726562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,255,0.2166208028793335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,511,0.3784463882446289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,511,0.3361167907714844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,1023,0.7017312049865723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,1023,0.535697603225708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,float16,2047,1.3627615928649903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,64,128,1,float16,fp8,2047,0.8893648147583008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,1,0.013339200615882873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,1,0.013675199449062347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,7,0.01340160071849823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,7,0.013519999384880067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,15,0.013502399623394012
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,15,0.013537600636482239
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,31,0.013388800621032714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,63,0.013443200290203095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,127,0.013321599364280701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,127,0.013281600177288055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,255,0.015174399316310882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,255,0.015340800583362579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,511,0.01895360052585602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,1023,0.020921599864959717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,1023,0.021352000534534454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,2047,0.039427199959754945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,2047,0.02759360074996948
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,1,0.013510400056838989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,1,0.01356319934129715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,3,0.01356160044670105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,3,0.013574400544166565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,7,0.01358720064163208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,15,0.0133775994181633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,31,0.0133200004696846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,31,0.013337600231170654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,63,0.013422399759292603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,63,0.013425600528717042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,127,0.013521599769592284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,127,0.013435199856758118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,3,0.013358399271965027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,255,0.015556800365447997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,3,0.013307200372219085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,511,0.018607999384403228
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,255,0.015547199547290802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,511,0.019180800020694732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,1023,0.03835679888725281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,1023,0.026542401313781737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,2047,0.060201597213745114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,1,0.08199359774589539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,3,0.07023839950561524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,3,0.08207679986953735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,7,0.07016000151634216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,7,0.08215519785881042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,15,0.07039679884910584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,15,0.08224800229072571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,31,0.07198079824447631
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,31,0.08166720271110535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,63,0.07608960270881653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,31,0.013415999710559845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,63,0.08334239721298217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,127,0.09235360026359558
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,255,0.11624159812927246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,255,0.11804319620132446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,511,0.20035998821258544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,float16,7,0.013344000279903411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,511,0.1760640025138855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,1023,0.2744672060012817
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,1023,0.36274080276489257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,2047,0.691596794128418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,1,0.01698399931192398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,fp8,63,0.013324800133705138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,1,0.017574399709701538
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,3,0.017528000473976135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,7,0.01720480024814606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,7,0.017660799622535705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,15,0.017504000663757326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,1,0.07073119878768921
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,15,0.01807200014591217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,31,0.017977599799633027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,31,0.017472000420093538
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,63,0.01743199974298477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,63,0.01793760061264038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,127,0.017689600586891174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,127,0.017955200374126436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,255,0.02163040041923523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,255,0.021911999583244322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,511,0.03903839886188507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,511,0.030481600761413576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,1023,0.06220160126686096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,1023,0.04691520035266876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,2047,0.104476797580719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,fp8,2047,0.07035840153694153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,1,0.24216480255126954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,fp8,2047,0.45730080604553225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,1,0.2984015941619873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,3,0.2394047975540161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,3,0.2986608028411865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,7,0.23950560092926027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,7,0.2986511945724487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,15,0.24533600807189943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,31,0.2544447898864746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,15,0.2993599891662598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,63,0.25838561058044435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,31,0.30593440532684324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,63,0.3165247917175293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,64,128,1,float16,float16,127,0.08014400005340576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,127,0.2633552074432373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,127,0.33271040916442873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,255,0.3933104038238525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,255,0.4178304195404053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,15,0.013380800187587739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,511,0.7435328006744385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,511,0.6550864219665528
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,float16,1023,1.3798432350158691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,64,128,1,float16,fp8,1023,1.0519647598266602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,1,0.5656816005706787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,7,0.5647840023040771
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,3,0.45926718711853026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,7,0.4689663887023926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,3,0.5689407825469971
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,15,0.484819221496582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,31,0.4863279819488525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,15,0.5676608085632324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,64,128,1,float16,float16,3,0.017235200107097625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,63,0.4897632122039795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,31,0.5941728115081787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,63,0.6035615921020507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,127,0.5089727878570557
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,127,0.6413407802581788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,1,0.9536687850952148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,1,1.115124797821045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,64,128,1,float16,fp8,2047,0.040996798872947694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,255,0.7526095867156982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,fp8,255,0.8273872375488281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,3,0.9636112213134765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,3,1.098680019378662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,7,0.9537455558776855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,15,0.9654959678649903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,7,1.1398880004882812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,15,1.175774383544922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,31,0.9650768280029297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,1,0.02537600100040436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,1,0.027143999934196472
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,3,0.025180798768997193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,3,0.027110400795936584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,63,0.9632320404052734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,7,0.025203201174736022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,31,1.178335952758789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,15,0.025278401374816895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,7,0.026256000995635985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,15,0.026280000805854797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,float16,127,1.001743984222412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,63,1.2138431549072266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,31,0.025246399641036987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,31,0.025945600867271424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,63,0.024281600117683412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,127,0.027268800139427184
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,63,0.026678401231765746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,127,0.02647840082645416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,255,0.04277600049972534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,255,0.03494719862937927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,64,128,1,float16,fp8,127,1.2414400100708007
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,64,128,1,float16,float16,511,0.018382400274276733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,511,0.06323519945144654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,511,0.05229920148849487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,1023,0.10604000091552734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,1023,0.0766431987285614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,fp8,2047,0.12310559749603271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,64,128,1,float16,float16,2047,0.18768320083618165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,3,1.8895824432373047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,1,1.9198944091796875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,7,1.851740837097168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,1,2.3151647567749025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,15,1.8669168472290039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,3,2.3538335800170898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,7,2.303878402709961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,31,1.8938352584838867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,float16,63,1.882761573791504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,15,2.3068096160888674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,31,2.2884992599487304
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,64,128,1,float16,fp8,63,2.30450553894043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,64,128,1,float16,float16,1,0.45888638496398926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,1,4.669249725341797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,1,4.578609466552734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,3,3.7380111694335936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,1,0.03880960047245026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,7,3.7162670135498046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,3,0.03885439932346344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,15,3.719776153564453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,3,4.605992126464844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,7,4.562862396240234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,7,0.03905439972877502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,15,0.0394463986158371
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,15,4.630859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,15,0.04386079907417297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,7,0.043479999899864195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,3,0.04383200109004974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,31,0.04390400052070618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,31,0.038815999031066896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,63,0.04078879952430725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,63,0.04389120042324066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,127,0.04678399860858917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,127,0.046079999208450316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,255,0.06798400282859803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,255,0.0637008011341095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,511,0.10878239870071411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,511,0.09224960207939148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,1023,0.19137760400772094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,1023,0.14121439456939697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,float16,2047,0.3553008079528809
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,64,128,1,float16,fp8,31,4.566088104248047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,2047,0.2298896074295044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,64,128,1,float16,float16,31,6.042036819458008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,64,128,1,float16,fp8,1,0.04463520050048828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,1,0.11776479482650756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,1,0.09817439913749695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,31,0.10178079605102539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,63,0.10645439624786376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,63,0.12583999633789061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,7,0.11844480037689209
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,127,0.11059520244598389
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,127,0.13381119966506957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,255,0.16323519945144654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,255,0.16796959638595582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,511,0.2908704042434692
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,511,0.2575135946273804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,1023,0.5334767818450927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,1023,0.4074831962585449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,15,0.11977440118789673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,7,0.09951679706573487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,2047,0.8861647605895996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,2047,1.0307120323181151
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,3,0.013352000713348388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,7,0.012995199859142303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,7,0.01327199935913086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,15,0.013091200590133667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,15,0.013247999548912048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,31,0.012902399897575379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,31,0.013052800297737121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,63,0.013051199913024902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,63,0.013236799836158752
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,127,0.012939199805259705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,127,0.01318880021572113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,255,0.014641599357128143
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,255,0.01507679969072342
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,511,0.017932799458503724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,511,0.01881439983844757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,1023,0.020798400044441223
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,1,0.012875199317932129
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,1023,0.024233600497245787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,2047,0.03160640001296997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,31,0.11974719762802125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,1,0.01311199963092804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,2047,0.039556801319122314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,1,0.01334560066461563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,3,0.013128000497817992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,7,0.013006399571895599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,3,0.013321599364280701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,7,0.013441599905490875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,15,0.013076800107955932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,31,0.013051199913024902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,15,0.013299199938774108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,63,0.012929600477218629
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,31,0.013278399407863618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,127,0.01329279989004135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,63,0.013363200426101684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,127,0.013369600474834441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,255,0.015142400562763215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,255,0.015379199385643005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,511,0.018080000579357148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,1023,0.029969599843025208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,511,0.01899999976158142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,1023,0.03610720038414002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,float16,2047,0.05342400074005127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,1,0.05517759919166565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,1,0.06422240138053895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,3,0.055201601982116696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,3,0.06408320069313049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,48,48,128,1,float16,fp8,2047,0.044198399782180785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,7,0.055550402402877806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,7,0.06416800022125244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,15,0.0553167998790741
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,15,0.06408479809761047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,31,0.0554639995098114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,31,0.06412479877471924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,3,0.09985759854316711
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,63,0.060759997367858885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,63,0.06484159827232361
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,127,0.07225919961929321
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,127,0.0643887996673584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,255,0.09523199796676636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,255,0.09211199879646301
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,511,0.15580639839172364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,511,0.13717119693756102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,fp8,3,0.11774400472640992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,1023,0.2787312030792236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,float16,2047,0.5249104022979736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,1,0.01669279932975769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,2047,0.34460160732269285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,3,0.01695519983768463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,1,0.017452800273895265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,7,0.01666879951953888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,3,0.017348800599575043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,7,0.017484800517559053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,15,0.01735839992761612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,31,0.016683200001716615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,31,0.017446400225162507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,63,0.016697600483894348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,63,0.017604799568653108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,127,0.016867199540138246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,127,0.01759680062532425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,255,0.020608000457286835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,255,0.021585600078105928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,511,0.029041600227355958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,511,0.03393760025501251
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,1023,0.052718400955200195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,1023,0.045335999131202696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,2047,0.08666239976882935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,fp8,2047,0.06917120218276977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,1,0.18371360301971434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,1,0.2236032009124756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,3,0.18341280221939088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,3,0.22352321147918702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,7,0.18343520164489746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,48,48,128,1,float16,fp8,1023,0.21034719944000244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,7,0.2236639976501465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,15,0.18486399650573732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,float16,3,0.0130048006772995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,15,0.22418720722198487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,31,0.1943168044090271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,31,0.2295151948928833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,63,0.19555039405822755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,63,0.23796958923339845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,127,0.203987193107605
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,127,0.24868640899658204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,255,0.3010591983795166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,255,0.3199824094772339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,511,0.5648111820220947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,1,0.3512048006057739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,511,0.48820319175720217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,3,0.3511023998260498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,1,0.4307407855987549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,float16,1023,1.1054224014282226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,48,48,128,1,float16,fp8,1023,0.78023681640625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,3,0.43155198097229003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,15,0.36624319553375245
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,7,0.4304512023925781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,31,0.3672895908355713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,7,0.35210399627685546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,15,0.43319201469421387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,31,0.457155179977417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,63,0.3706464052200317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,127,0.38718719482421876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,63,0.45030879974365234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,127,0.4917471885681152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,float16,255,0.5745183944702148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,1,0.6967840194702148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,1,0.8439423561096191
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,48,48,128,1,float16,float16,15,0.09979199767112731
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,48,48,128,1,float16,fp8,255,0.6143375873565674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,3,0.7517055988311767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,48,48,128,1,float16,fp8,1,0.013035200536251068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,3,0.846678352355957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,7,0.7191199779510498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,7,0.8330944061279297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,15,0.7316160202026367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,15,0.8765744209289551
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,1,0.02040639966726303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,31,1.1062095642089844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,63,0.7261775970458985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,31,0.9121184349060059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,1,0.021631999313831328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,63,0.902400016784668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,float16,127,0.7491744041442872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,7,0.021091200411319733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,7,0.02173600047826767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,3,0.02050720006227493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,31,0.02120800018310547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,15,0.02017119973897934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,63,0.022417600452899932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,31,0.021817600727081297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,15,0.02163040041923523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,3,0.021670399606227873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,63,0.020606400072574617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,255,0.027735999226570128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,127,0.022537599503993987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,255,0.03487679958343506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,127,0.020476800203323365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,511,0.050256001949310306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,511,0.041756799817085265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,1023,0.08356480002403259
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,1023,0.06312479972839355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,float16,2047,0.14500160217285157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,48,48,128,1,float16,fp8,2047,0.09538400173187256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,3,1.4323375701904297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,7,1.3940336227416992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,1,1.8083551406860352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,1,1.7318143844604492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,3,1.7323616027832032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,48,48,128,1,float16,float16,15,0.016755199432373045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,15,1.409772777557373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,48,48,128,1,float16,fp8,127,0.9495247840881348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,31,1.4223360061645507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,7,1.792478370666504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,31,1.742843246459961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,float16,63,1.4363183975219727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,15,2.760763168334961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,48,48,128,1,float16,fp8,63,1.7438272476196288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,1,2.8301824569702148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,3,2.7832624435424806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,1,3.4525215148925783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,7,2.8198944091796876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,1,0.03105120062828064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,3,0.03105599880218506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,3,3.4538177490234374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,1,0.03516480028629303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,3,0.03463680148124695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,7,0.030878400802612303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,7,0.035257598757743834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,15,0.031137600541114807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,15,0.034683200716972354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,31,0.03182399868965149
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,31,0.03476319909095764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,63,0.031145599484443665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,63,0.03524799942970276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,127,0.03813279867172241
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,127,0.03474079966545105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,255,0.05639520287513733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,15,2.7720256805419923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,255,0.049795201420783995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,511,0.08586239814758301
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,511,0.07266079783439636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,1023,0.14824800491333007
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,1023,0.10898079872131347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,fp8,2047,0.1775936007499695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,48,48,128,1,float16,float16,2047,0.27118399143218996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,15,3.4917022705078127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,48,48,128,1,float16,float16,31,2.8185632705688475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,31,3.4252639770507813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,48,48,128,1,float16,fp8,7,5.473819351196289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,1,0.08347200155258179
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,1,0.0988864004611969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,31,0.08457599878311158
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,31,0.09898239970207215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,63,0.08983520269393921
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,63,0.10571680068969727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,127,0.09405760169029236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,127,0.11315360069274902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,255,0.13992320299148558
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,255,0.14321919679641723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,511,0.24434878826141357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,511,0.21628000736236572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,1023,0.44554882049560546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,1023,0.339190411567688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,15,0.08387839794158936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,1,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,2047,0.8600111961364746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,2047,0.5649727821350098
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,3,0.012563200294971466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,3,0.013463999330997466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,15,0.10085599422454834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,7,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,7,0.013415999710559845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,3,0.08453119993209839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,3,0.10160000324249267
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,31,0.013593600690364837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,63,0.013672000169754029
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,127,0.01287200003862381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,127,0.013600000739097595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,63,0.012750400602817536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,255,0.01478399932384491
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,255,0.015334400534629821
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,511,0.01762399971485138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,511,0.01905599981546402
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,1023,0.018644799292087556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,1023,0.01984799951314926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,2047,0.027142399549484254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,2047,0.023798400163650514
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,1,0.012934400141239167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,3,0.012854400277137756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,1,0.01356000006198883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,3,0.013468800485134125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,7,0.012838399410247803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,7,0.013558399677276612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,15,0.012824000418186187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,15,0.013372799754142762
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,31,0.012761600315570831
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,31,0.013500800728797913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,63,0.01292479932308197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,63,0.013531200587749481
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,127,0.012878400087356568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,127,0.013625599443912506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,255,0.014798399806022645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,255,0.015556800365447997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,511,0.017932799458503724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,511,0.019198399782180787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,1023,0.031035199761390686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,1023,0.030099201202392577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,float16,2047,0.051963198184967044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,40,40,128,1,float16,fp8,2047,0.04521600008010864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,1,0.047625601291656494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,1,0.055307197570800784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,3,0.047111999988555905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,15,0.01282079964876175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,3,0.05487040281295776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,7,0.04701279997825623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,7,0.054923200607299806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,1,0.013395200669765472
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,15,0.04701119959354401
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,15,0.05483199954032898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,31,0.04715520143508911
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,31,0.05493599772453308
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,63,0.0515824019908905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,63,0.05506399869918823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,127,0.05570399761199951
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,127,0.06190559864044189
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,255,0.08215839862823486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,255,0.07930880188941955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,511,0.1322543978691101
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,511,0.11630719900131226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,1023,0.23483679294586182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,1023,0.17912319898605347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,1,0.016444799304008485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,1,0.017716799676418305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,3,0.01632159948348999
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,3,0.0177279993891716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,float16,2047,0.4376704216003418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,40,40,128,1,float16,fp8,2047,0.29766240119934084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,7,0.01627040058374405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,15,0.016363200545310975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,31,0.01642400026321411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,7,0.017683200538158417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,15,0.017977599799633027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,31,0.017679999768733978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,63,0.01643519997596741
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,63,0.017987200617790224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,127,0.01655679941177368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,127,0.017769600450992584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,255,0.020377600193023683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,255,0.021822400391101837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,511,0.02863680124282837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,511,0.029398399591445922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,1023,0.05079360008239746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,1023,0.04573279917240143
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,float16,2047,0.08570240139961242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,40,40,128,1,float16,fp8,2047,0.07007520198822022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,1,0.15404800176620484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,3,0.15534080266952516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,1,0.18993760347366334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,3,0.188046395778656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,7,0.15707999467849731
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,7,0.18852959871292113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,15,0.15461119413375854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,15,0.18995519876480102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,31,0.16363359689712526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,31,0.18888959884643555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,63,0.1666815996170044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,63,0.20165600776672363
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,127,0.1718127965927124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,127,0.2100752115249634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,255,0.257806396484375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,255,0.2735599994659424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,511,0.5164688110351563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,511,0.4084144115447998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,1,0.29256319999694824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,fp8,1023,0.6521279811859131
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,1,0.363102388381958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,40,40,128,1,float16,float16,1023,0.8700672149658203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,3,0.293667197227478
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,7,0.2925424098968506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,3,0.36392319202423096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,float16,31,0.01271039992570877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,7,0.3691855907440186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,40,40,128,1,float16,fp8,15,0.01345440000295639
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,31,0.3103408098220825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,15,0.3680639982223511
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,15,0.359279990196228
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,31,0.3884912014007568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,63,0.3173327922821045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,63,0.39145119190216066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,127,0.3255376100540161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,float16,255,0.48479838371276857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,127,0.6107103824615479
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,40,40,128,1,float16,fp8,255,0.5256048202514648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,1,0.5830224037170411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,1,0.7296527862548828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,3,0.5784304141998291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,15,0.613486385345459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,3,0.7095344066619873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,7,0.7234303951263428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,7,0.904599952697754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,31,0.6021183967590332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,15,0.7390575885772706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,31,0.7452159881591797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,63,0.6192512035369873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,63,0.8882608413696289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,1,0.01998399943113327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,3,0.022462399303913118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,3,0.02171359956264496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,1,0.021852800250053407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,float16,127,0.6355984210968018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,7,0.020151999592781068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,40,40,128,1,float16,fp8,127,0.7990880012512207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,15,0.01993280053138733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,7,0.02173279970884323
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,63,0.020108799636363982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,31,0.020267200469970704
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,15,0.02178879976272583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,127,0.020326399803161622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,127,0.022168000042438508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,31,0.02200479954481125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,63,0.021798400580883025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,255,0.02851040065288544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,511,0.04141440093517303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,511,0.045414400100708005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,255,0.02783679962158203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,1023,0.07429919838905334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,1023,0.06264160275459289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,fp8,2047,0.09739840030670166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,40,40,128,1,float16,float16,2047,0.12999360561370848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,1,1.1855567932128905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,3,1.1703840255737306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,1,1.3838159561157226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,3,1.4121503829956055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,float16,7,0.08439199924468994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,7,2.1413040161132812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,40,40,128,1,float16,fp8,7,0.1000864028930664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,7,1.4588080406188966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,15,1.1768704414367677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,31,1.176363182067871
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,15,1.4505279541015625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,63,1.481822395324707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,float16,63,2.070449638366699
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,1,2.3558000564575194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,3,2.335638427734375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,1,2.9029823303222657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,7,2.3254720687866213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,1,0.027198401093482972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,1,0.030744001269340515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,3,2.868534469604492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,3,0.027422401309013366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,3,0.03068479895591736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,7,0.027438399195671082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,7,2.919550323486328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,15,0.027555200457572936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,7,0.030579200387001036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,15,0.03076159954071045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,31,2.3170976638793945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,31,0.02754879891872406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,31,0.03072800040245056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,63,0.027611199021339416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,127,0.03239200115203857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,63,0.030534398555755616
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,127,0.030817601084709167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,15,2.872982406616211
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,255,0.04915040135383606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,255,0.0435696005821228
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,511,0.062063997983932494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,511,0.07354879975318909
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,1023,0.12679519653320312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,1023,0.09388800263404846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,40,40,128,1,float16,float16,15,3.9198543548583986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,1,0.06704639792442321
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,3,0.06731200218200684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,float16,2047,0.2284912109375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,7,0.0671567976474762
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,40,40,128,1,float16,fp8,31,2.860204887390137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,40,40,128,1,float16,fp8,2047,0.15069279670715333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,15,0.06766080260276794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,63,0.07349119782447815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,31,0.06797119975090027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,127,0.07759360074996949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,1,0.0786288022994995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,40,40,128,1,float16,fp8,31,1.4946607589721679
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,255,0.11809600591659546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,511,0.2007551908493042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,511,0.17428799867630004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,1023,0.36256000995635984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,1023,0.27180640697479247
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,15,0.08128479719161988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,2047,0.6904143810272216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,2047,0.4545487880706787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,float16,4095,1.3618063926696777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,4095,0.8551535606384277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,1,0.012936000525951386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,1,0.012559999525547028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,3,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,3,0.01287200003862381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,31,0.082915198802948
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,7,0.012531200051307678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,15,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,7,0.013040000200271606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,15,0.012884800136089326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,31,0.012572799623012543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,31,0.012964800000190735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,63,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,63,0.012804800271987915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,127,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,127,0.013006399571895599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,255,0.014291200041770934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,255,0.014910399913787842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,511,0.017377600073814392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,511,0.01852799952030182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,1023,0.018228800594806673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,1023,0.017574399709701538
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,2047,0.0201664000749588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,2047,0.021432000398635864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,float16,4095,0.03739199936389923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,32,128,1,float16,fp8,4095,0.027419200539588927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,1,0.0126351997256279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,1,0.013043199479579926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,3,0.01271039992570877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,3,0.013105599582195282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,7,0.012615999579429627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,7,0.013059200346469879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,15,0.01266240030527115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,15,0.013023999333381654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,31,0.012747199833393097
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,63,0.012630400061607362
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,31,0.012943999469280243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,63,0.013208000361919403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,127,0.012561599910259246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,127,0.013209599256515502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,255,0.014351999759674073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,255,0.015035200119018554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,511,0.01733279973268509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,511,0.01858240067958832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,1023,0.021459199488162994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,1023,0.02115679979324341
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,2047,0.037176001071929934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,2047,0.028009599447250365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,float16,4095,0.059008002281188965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,32,128,1,float16,fp8,4095,0.04276959896087647
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,1,0.03983680009841919
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,1,0.04552479982376099
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,3,0.03987680077552795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,3,0.04555040001869202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,7,0.03980959951877594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,7,0.04534400105476379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,15,0.03986720144748688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,15,0.045289599895477296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,63,0.0839631974697113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,31,0.03988640010356903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,31,0.045300799608230594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,63,0.04046559929847717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,63,0.04529919922351837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,127,0.047225600481033324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,127,0.046700799465179445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,255,0.06830400228500366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,255,0.06394720077514648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,511,0.10872000455856323
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,511,0.09338719844818115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,1023,0.14422240257263183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,1023,0.19111039638519287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,2047,0.3539167881011963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,2047,0.23710238933563232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,1,0.012608000636100769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,1,0.013041600584983826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,3,0.01265760064125061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,float16,4095,0.6857855796813965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,3,0.01316159963607788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,32,128,1,float16,fp8,4095,0.43266878128051756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,7,0.012675200402736665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,15,0.01316159963607788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,7,0.013064000010490417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,15,0.012596799433231354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,63,0.01257600039243698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,31,0.012937599420547485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,31,0.01268640011548996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,63,0.01329440027475357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,127,0.01268479973077774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,255,0.014843200147151948
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,127,0.013062399625778199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,255,0.015192000567913056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,511,0.018719999492168425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,511,0.018859200179576874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,1023,0.03643360137939453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,1023,0.026807999610900878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,2047,0.058396798372268674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,127,0.09330559968948364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,2047,0.04146080017089844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,float16,4095,0.10081280469894409
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,32,128,1,float16,fp8,4095,0.0661952018737793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,1,0.12579519748687745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,1,0.15487840175628662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,3,0.12776319980621337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,3,0.15538879632949829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,7,0.12523360252380372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,15,0.12761919498443602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,7,0.1539471983909607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,15,0.15583839416503906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,31,0.13439680337905885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,31,0.1545567989349365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,63,0.13512639999389647
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,127,0.14181920289993286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,63,0.1660431981086731
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,127,0.17411999702453612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,255,0.21065919399261473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,255,0.2179919958114624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,511,0.39008159637451173
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,511,0.3350608110427856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,1023,0.7036367893218994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,1023,0.5303791999816895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,1,0.23912160396575927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,1,0.2902127981185913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,3,0.2391632080078125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,fp8,2047,0.8913583755493164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,32,128,1,float16,float16,2047,1.3596575736999512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,3,0.29009280204772947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,7,0.23938560485839844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,15,0.29085440635681153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,31,0.25369439125061033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,7,0.2919663906097412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,15,0.2624207973480225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,31,0.3015343904495239
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,63,0.2561216115951538
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,127,0.27563838958740233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,63,0.3149967908859253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,127,0.3252640008926392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,255,0.3944767951965332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,3,0.08194559812545776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,255,0.4319151878356934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,float16,511,0.7672463893890381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,1,0.4644415855407715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,1,0.5730144023895264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,3,0.4758175849914551
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,32,128,1,float16,fp8,511,0.6414271831512451
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,7,0.4734032154083252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,7,0.573140811920166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,15,0.49661760330200194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,3,0.579315185546875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,15,0.5681424140930176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,31,0.4922031879425049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,63,0.49815998077392576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,31,0.6702623844146729
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,63,0.6096240043640136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,127,0.5071792125701904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,1,0.01687999963760376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,1,0.01759359985589981
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,3,0.017723199725151063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,7,0.016769599914550782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,127,0.6474448204040527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,3,0.0162432000041008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,15,0.016791999340057373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,7,0.017660799622535705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,31,0.016279999911785126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,31,0.01780479997396469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,63,0.016728000342845918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,15,0.017870399355888366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,float16,255,0.7604671955108643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,63,0.017324799299240114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,127,0.017110399901866913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,127,0.017287999391555786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,255,0.020849600434303284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,511,0.036636799573898315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,511,0.02972480058670044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,32,128,1,float16,fp8,255,0.8209775924682617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,255,0.021160000562667848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,1023,0.059832000732421876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,4095,0.1855679988861084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,float16,2047,0.10152959823608398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,1023,0.04578559994697571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,4095,0.11813759803771973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,32,128,1,float16,fp8,2047,0.06818879842758178
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,1,1.5068320274353026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,3,0.9391087532043457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,1,1.137507152557373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,3,1.1052703857421875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,7,0.9504672050476074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,7,1.1414992332458496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,31,0.9775535583496093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,15,1.1577391624450684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,63,0.9618144035339355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,31,1.1854127883911132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,7,0.08200479745864868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,15,1.8041839599609375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,63,1.1611151695251465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,float16,127,1.012110424041748
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,32,128,1,float16,fp8,127,1.23056001663208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,1,1.8826559066772461
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,3,1.8996496200561523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,7,1.8697807312011718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,1,2.3025999069213867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,15,1.861427116394043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,7,2.3369855880737305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,31,1.9060752868652344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,15,2.285367965698242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,float16,63,1.8857536315917969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,3,4.548728179931641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,31,2.2881839752197264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,3,0.023455999791622162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,7,0.023579199612140656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,1,0.0259552001953125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,1,0.02324319928884506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,7,0.02555519938468933
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,3,0.026311999559402464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,15,0.023639999330043793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,15,0.02563520073890686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,31,0.023636800050735474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,63,0.025747200846672057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,63,0.02407519966363907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,31,0.025726398825645445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,127,0.025177600979804992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,127,0.02587519884109497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,255,0.041124799847602846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,255,0.035148799419403076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,511,0.06113920211791992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,511,0.05059679746627808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,1023,0.10369759798049927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,1023,0.07607679963111877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,2047,0.12222720384597778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,1,0.05287520289421081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,2047,0.23163681030273436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,3,0.05285279750823975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,7,0.05285279750823975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,fp8,4095,0.21939520835876464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,32,128,1,float16,float16,4095,0.3517888069152832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,32,128,1,float16,fp8,63,2.2753616333007813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,15,0.05288640260696411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,31,0.053016000986099245
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,63,0.05750399827957153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,32,128,1,float16,fp8,255,0.11829119920730591
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,127,0.06408320069313049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,1,0.06283360123634338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,255,0.0943008005619049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,255,0.09092479944229126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,511,0.1342560052871704
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,511,0.1557279944419861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,1023,0.2801487922668457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,2047,0.5227615833282471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,2047,0.3438240051269531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,7,0.06365119814872741
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,float16,4095,1.0235887527465821
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,1,0.012603199481964112
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,1,0.013299199938774108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,3,0.012598399817943574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,3,0.013052800297737121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,4095,0.6409327983856201
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,7,0.012356799840927125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,15,0.01228799968957901
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,7,0.012777599692344665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,15,0.013067199289798737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,31,0.013035200536251068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,31,0.012127999961376191
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,63,0.012329600006341934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,63,0.01284160017967224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,127,0.012931199371814727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,127,0.012225600332021714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,255,0.01387840062379837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,255,0.014892800152301789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,511,0.016865600645542145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,511,0.01839679926633835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,1023,0.01647839993238449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,1023,0.017323200404644013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,2047,0.019908800721168518
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,2047,0.01945440024137497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,fp8,4095,0.025377601385116577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,24,24,128,1,float16,float16,4095,0.03168320059776306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,1,0.012145599722862244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,1,0.013327999413013459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,3,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,3,0.012919999659061432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,7,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,7,0.012950399518013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,15,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,15,0.012876799702644348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,31,0.01228799968957901
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,15,0.06466079950332641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,31,0.012923200428485871
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,63,0.012191999703645706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,127,0.01225920021533966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,63,0.013257600367069244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,127,0.013212800025939941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,255,0.014079999923706055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,255,0.015177600085735321
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,511,0.017156800627708434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,511,0.018403199315071107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,1023,0.02367520034313202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,1023,0.019894400238990785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,2047,0.03791840076446533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,2047,0.031159999966621398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,float16,4095,0.061913597583770755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,24,24,128,1,float16,fp8,4095,0.052590399980545044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,1,0.03193120062351227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,1,0.03673279881477356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,3,0.031553599238395694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,3,0.03636800050735474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,7,0.032227200269699094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,7,0.03587839901447296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,15,0.031856000423431396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,15,0.03598560094833374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,31,0.03188480138778686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,31,0.03597759902477264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,63,0.032339200377464294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,63,0.03592959940433502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,127,0.03819999992847443
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,127,0.03703039884567261
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,255,0.056148797273635864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,255,0.05121279954910278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,511,0.08502879738807678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,511,0.0728048026561737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,1023,0.14867520332336426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,1023,0.11175359487533569
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,2047,0.2705696105957031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,2047,0.17996959686279296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,31,0.06462399959564209
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,1,0.012392000108957291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,1,0.013230399787425995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,3,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,3,0.013214400410652161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,fp8,4095,0.3344880104064941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,7,0.01239359974861145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,24,24,128,1,float16,float16,4095,0.5194159984588623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,15,0.012537600100040435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,7,0.013337600231170654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,31,0.012316799908876418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,31,0.01343040019273758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,15,0.013152000308036805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,63,0.012464000284671784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,63,0.013307200372219085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,127,0.01329600065946579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,127,0.012596799433231354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,255,0.015497599542140961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,255,0.014604799449443817
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,511,0.017585599422454835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,511,0.018995200097560883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,1023,0.034755200147628784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,1023,0.029710400104522704
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,2047,0.05161920189857483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,2047,0.045947200059890746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,fp8,4095,0.07065600156784058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,24,24,128,1,float16,float16,4095,0.0857583999633789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,1,0.0998207986354828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,3,0.09861279726028442
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,1,0.12038400173187255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,3,0.12076959609985352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,7,0.0986735999584198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,7,0.12056800127029418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,15,0.100108802318573
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,15,0.1197167992591858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,31,0.10163359642028809
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,31,0.12112640142440796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,63,0.10496480464935302
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,63,0.1280608057975769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,127,0.11104799509048462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,127,0.13503199815750122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,255,0.17217600345611572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,255,0.16896480321884155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,511,0.29632959365844724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,511,0.2608511924743652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,63,0.06455839872360229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,1023,0.5331215858459473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,1023,0.40609121322631836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,1,0.18086880445480347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,3,0.18403680324554444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,1,0.22438559532165528
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,3,0.22171199321746826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,7,0.18458240032196044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,7,0.22423839569091797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,float16,2047,1.0271984100341798
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,24,24,128,1,float16,fp8,2047,0.6815248012542725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,15,0.18250240087509156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,31,0.19412000179290773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,15,0.22664799690246581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,31,0.22980639934539795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,63,0.2355679988861084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,63,0.1945695996284485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,127,0.20606560707092286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,127,0.25404160022735595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,255,0.3089967966079712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,255,0.3167023897171021
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,1,0.3503983974456787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,float16,511,0.5810207843780517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,24,24,128,1,float16,fp8,511,0.49300642013549806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,3,0.3490623950958252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,1,0.43480958938598635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,3,0.43037919998168944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,7,0.3513360023498535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,15,0.3674304008483887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,15,0.43534398078918457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,7,0.43343358039855956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,31,0.37124640941619874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,31,0.45671520233154295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,63,0.3732719898223877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,127,0.3926559925079346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,127,0.07305279970169068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,63,0.45616960525512695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,127,0.4891488075256348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,1,0.01753759980201721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,1,0.016305600106716157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,3,0.01602399945259094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,3,0.01746560037136078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,float16,255,0.5808944225311279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,7,0.017136000096797943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,7,0.015812799334526062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,15,0.01619199961423874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,15,0.01717280000448227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,31,0.015960000455379486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,63,0.016172799468040466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,63,0.01723040044307709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,127,0.016051200032234193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,24,24,128,1,float16,fp8,255,0.6180480003356934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,31,0.017611199617385866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,127,0.017500799894332886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,255,0.019568000733852387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,255,0.02099519968032837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,511,0.03359679877758026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,1023,0.05118719935417175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,511,0.028803199529647827
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,1023,0.04616000056266785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,2047,0.08429279923439026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,2047,0.06864320039749146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,fp8,4095,0.12191200256347656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,24,24,128,1,float16,float16,4095,0.15322240591049194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,1,0.6989439964294434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,3,0.7027743816375732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,1,0.82750244140625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,7,0.724724817276001
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,3,0.8367839813232422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,15,0.7181392192840577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,7,0.8332048416137695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,15,0.8803584098815918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,31,0.7288352012634277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,63,0.7204880237579345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,31,0.8987664222717285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,float16,127,0.7542175769805908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,63,0.8779727935791015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,24,24,128,1,float16,fp8,127,0.9410655975341797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,1,1.398961639404297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,3,1.404520034790039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,1,1.7156784057617187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,1023,0.20953280925750734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,3,1.7222303390502929
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,7,1.4135824203491212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,15,1.3998895645141602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,1,0.019438399374485014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,1,0.021687999367713928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,7,1.7759391784667968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,31,1.4211104393005372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,15,1.7294591903686523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,float16,63,1.4140704154968262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,31,1.7527664184570313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,7,0.019968000054359437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,3,0.019782400131225585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,15,0.019414399564266206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,31,0.020127999782562255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,7,0.02173600047826767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,63,0.02171040028333664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,31,0.021631999313831328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,24,24,128,1,float16,fp8,63,1.7317823410034179
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,127,0.02163359969854355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,15,0.021886399388313292
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,63,0.019755199551582336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,3,0.02186879962682724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,127,0.019827200472354888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,511,0.04856959879398346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,255,0.027327999472618103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,255,0.03305279910564422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,511,0.041387200355529785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,1023,0.06071839928627014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,1023,0.08196480274200439
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,1,0.038812801241874695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,2047,0.14303840398788453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,2047,0.09514560103416443
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,3,0.03834559917449951
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,7,0.03853119909763336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,15,0.03830400109291077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,float16,4095,0.2679647922515869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,24,24,128,1,float16,fp8,4095,0.16960320472717286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,31,0.03875359892845154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,63,0.0390720009803772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,127,0.04656000137329101
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,24,24,128,1,float16,fp8,3,0.06412960290908813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,255,0.06935999989509582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,1,0.045398399233818054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,511,0.11096160411834717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,511,0.09351840019226074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,1023,0.19424159526824952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,1023,0.1421615958213806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,2047,0.3595855951309204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,2047,0.2373136043548584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,3,0.04542239904403687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,4095,0.6878399848937988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,4095,0.4322048187255859
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,8191,0.8351887702941895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,float16,8191,1.3537343978881835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,1,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,3,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,1,0.013308799266815186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,3,0.013505600392818451
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,7,0.04594559967517853
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,7,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,15,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,7,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,15,0.013014400005340576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,31,0.011947199702262878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,31,0.012971200048923492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,63,0.011828800290822982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,63,0.012921600043773651
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,127,0.011935999989509583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,127,0.01284160017967224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,255,0.013412800431251527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,255,0.015131199359893798
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,511,0.01639840006828308
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,511,0.01884479969739914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,1023,0.016406400501728056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,1023,0.016969600319862367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,2047,0.01870879977941513
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,2047,0.01780160069465637
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,4095,0.022609600424766542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,4095,0.021745599806308746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,float16,8191,0.03770560026168823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,16,128,1,float16,fp8,8191,0.02794719934463501
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,1,0.012731200456619263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,1,0.012275200337171555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,3,0.012118399888277055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,3,0.012895999848842621
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,7,0.012111999839544297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,7,0.01282079964876175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,15,0.012091200053691863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,15,0.01290079951286316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,31,0.0121568001806736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,31,0.012943999469280243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,63,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,63,0.012753599882125854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,127,0.012163200229406358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,127,0.012928000092506409
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,255,0.014091199636459351
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,255,0.01462240070104599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,511,0.01725119948387146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,511,0.018467199802398682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,1023,0.018003199994564057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,1023,0.017529599368572235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,2047,0.0204815998673439
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,2047,0.021352000534534454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,4095,0.02762399911880493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,4095,0.03684639930725098
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,float16,8191,0.05894399881362915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,16,128,1,float16,fp8,8191,0.04176799952983856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,1,0.0245728000998497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,1,0.027614399790763855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,3,0.024264000356197357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,3,0.02755360007286072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,7,0.02418559938669205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,7,0.027551999688148497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,15,0.024566400051116943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,15,0.027588799595832825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,31,0.024505600333213806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,31,0.02764959931373596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,63,0.02454719990491867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,63,0.04595519900321961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,63,0.027651199698448183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,127,0.025319999456405638
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,127,0.027055999636650084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,255,0.035390400886535646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,255,0.042208001017570496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,511,0.061577600240707395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,511,0.051630401611328126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,1023,0.10628800392150879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,1023,0.07819200158119202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,2047,0.12596160173416138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,2047,0.18841919898986817
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,4095,0.35358080863952634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,4095,0.22628960609436036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,1,0.012825599312782288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,1,0.01210239976644516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,3,0.012283200025558471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,3,0.012873600423336028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,7,0.012188799679279327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,7,0.012804800271987915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,fp8,8191,0.4206672191619873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,15,0.01231359988451004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,15,0.012863999605178833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,16,128,1,float16,float16,8191,0.6871776103973388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,31,0.01284479945898056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,63,0.011980800330638886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,31,0.012091200053691863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,127,0.01236959993839264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,63,0.012961600720882416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,127,0.012579199671745301
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,255,0.013777600228786468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,255,0.014897599816322327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,511,0.018187199532985688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,511,0.017127999663352968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,1023,0.02032800018787384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,1023,0.02093279957771301
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,2047,0.03697440028190613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,2047,0.0272816002368927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,4095,0.058524799346923825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,4095,0.0416047990322113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,fp8,8191,0.06637600064277649
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,1,0.07027999758720398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,16,128,1,float16,float16,8191,0.10145440101623535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,3,0.06998239755630493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,1,0.08280640244483947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,3,0.08230400085449219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,127,0.04689919948577881
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,7,0.07027519941329956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,7,0.08280320167541504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,15,0.0691872000694275
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,15,0.08340799808502197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,31,0.07090719938278198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,31,0.08243039846420289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,63,0.07476639747619629
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,63,0.08597599864006042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,127,0.0821183979511261
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,127,0.09307039976119995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,255,0.12380000352859497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,255,0.1190991997718811
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,511,0.20723841190338135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,511,0.17504960298538208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,1023,0.2772079944610596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,1023,0.3644095897674561
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,2047,0.4584959983825684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,2047,0.6975408077239991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,1,0.15423519611358644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,3,0.12700639963150023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,1,0.12526880502700805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,7,0.12564480304718018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,3,0.15451040267944335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,fp8,4095,0.8578288078308105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,7,0.1527392029762268
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,15,0.12725600004196166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,31,0.13068799972534179
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,31,0.1540127992630005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,15,0.15482079982757568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,16,128,1,float16,float16,4095,1.364863967895508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,63,0.13719040155410767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,63,0.1666543960571289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,127,0.1476207971572876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,255,0.21829440593719482
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,255,0.22120959758758546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,127,0.18729439973831177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,511,0.3998271942138672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,511,0.34100799560546874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,1,0.24111518859863282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,1,0.29427199363708495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,fp8,1023,0.5405983924865723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,255,0.06455680131912231
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,16,128,1,float16,float16,1023,0.708025598526001
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,3,0.2412800073623657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,3,0.2940736055374146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,7,0.24355039596557618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,7,0.29216799736022947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,15,0.2940943956375122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,15,0.25182878971099854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,31,0.25671360492706297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,63,0.2564768075942993
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,31,0.3140144109725952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,63,0.31480319499969484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,127,0.27215359210968015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,127,0.32943999767303467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,1,0.012332800030708312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,255,0.4024943828582764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,1,0.012999999523162841
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,255,0.42552962303161623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,3,0.013020800054073333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,3,0.012625600397586822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,7,0.013048000633716583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,7,0.01271200031042099
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,15,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,15,0.013009600341320038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,31,0.012673600018024445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,31,0.01313920021057129
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,63,0.012110400199890136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,63,0.013223999738693237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,127,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,127,0.01265919953584671
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,255,0.014614400267601014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,255,0.014883199334144592
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,511,0.017632000148296356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,float16,511,0.7831615924835205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,16,128,1,float16,fp8,511,0.6604832172393799
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,511,0.01887679994106293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,1023,0.025775998830795288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,1023,0.036236798763275145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,2047,0.04156000018119812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,4095,0.10037440061569214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,2047,0.06518080234527587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,4095,0.06502400040626526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,float16,8191,0.18404480218887329
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,16,128,1,float16,fp8,8191,0.11786079406738281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,1,0.4687136173248291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,3,0.4623871803283691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,1,0.5644495964050293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,7,0.4787136077880859
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,3,0.5588031768798828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,15,0.49254398345947265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,31,0.4921311855316162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,7,0.5673647880554199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,15,0.5718031883239746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,63,0.49489121437072753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,31,0.6108160018920898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,63,0.6095168113708496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,127,0.5174767971038818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,127,0.6271632194519043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,1,0.952724838256836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,fp8,255,0.8143520355224609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,3,0.9480192184448242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,1,1.099131202697754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,16,128,1,float16,float16,255,1.3973456382751466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,15,0.04631040096282959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,3,1.1171248435974122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,7,0.945406436920166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,15,0.948084831237793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,7,1.1365407943725585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,31,0.963815975189209
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,15,1.4278047561645508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,1,0.030780801177024843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,63,0.9628447532653809
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,1,0.017947199940681457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,31,1.1593152046203614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,3,0.01589920073747635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,3,0.017932799458503724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,7,0.015902400016784668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,7,0.01725279986858368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,15,0.016760000586509706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,15,0.01687840074300766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,31,0.017990399897098542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,31,0.01606079936027527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,63,0.015887999534606935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,63,1.1702816009521484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,63,0.017975999414920805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,float16,127,1.0073871612548828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,127,0.01653279960155487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,127,0.017022399604320525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,255,0.01966080069541931
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,255,0.02208160012960434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,511,0.02855519950389862
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,1023,0.06003519892692566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,1023,0.04359680116176605
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,511,0.039987200498580934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,2047,0.10318880081176758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,2047,0.06772639751434326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,1,0.03140000104904175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,4095,0.11635680198669433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,4095,0.1849776029586792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,16,128,1,float16,fp8,127,1.2180656433105468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,3,0.03156639933586121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,fp8,8191,0.2855151891708374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,16,128,1,float16,float16,8191,0.35250239372253417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,7,0.032179200649261476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,15,0.03264159858226776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,31,0.03221279978752136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,63,0.031814399361610415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,127,0.04003520011901855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,1,0.0363072007894516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,255,0.05827999711036682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,16,128,1,float16,fp8,31,0.04612640142440796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,255,0.05182240009307861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,511,0.08768159747123719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,511,0.0736735999584198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,1023,0.1527184009552002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,1023,0.11149280071258545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,2047,0.18254879713058472
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,2047,0.27425599098205566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,4095,0.5212800025939941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,31,0.03734239935874939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,float16,8191,1.0194111824035645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,1,0.011820799857378005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,1,0.012641599774360657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,8191,0.9302319526672364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,3,0.011937599629163742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,7,0.011929599940776825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,3,0.012860800325870513
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,7,0.012756800651550293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,15,0.012135999649763108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,15,0.0126351997256279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,31,0.012187200039625168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,31,0.012785600125789642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,63,0.012127999961376191
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,63,0.01265760064125061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,127,0.012089599668979645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,127,0.012615999579429627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,255,0.013784000277519226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,255,0.014643199741840363
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,511,0.016707199811935424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,511,0.018539200723171233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,1023,0.01737920045852661
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,2047,0.017652800679206847
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,1023,0.018273599445819855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,2047,0.018423999845981597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,4095,0.020657600462436677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,4095,0.020337599515914916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,float16,8191,0.03313120007514954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,12,12,128,1,float16,fp8,8191,0.026104000210762025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,1,0.01234399974346161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,1,0.012647999823093415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,3,0.012318400293588638
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,3,0.012995199859142303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,7,0.012291199713945388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,7,0.013051199913024902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,15,0.012265600264072418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,15,0.013012799620628356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,31,0.01226079985499382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,31,0.012993599474430084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,63,0.012249600142240524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,63,0.013126400113105775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,63,0.03707360029220581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,127,0.012220799922943115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,255,0.01404159963130951
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,127,0.012947200238704682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,255,0.015065599977970124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,511,0.016996799409389494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,511,0.018404799699783325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,1023,0.01733600050210953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,1023,0.018353599309921264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,2047,0.02078240066766739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,2047,0.020535999536514284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,4095,0.032846400141716005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,4095,0.026492801308631898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,float16,8191,0.050241601467132566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,12,12,128,1,float16,fp8,8191,0.039529600739479066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,1,0.02067680060863495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,1,0.022915199398994446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,3,0.0207056000828743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,3,0.022932800650596618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,7,0.020771199464797975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,7,0.022932800650596618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,15,0.020713600516319274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,15,0.0230335995554924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,31,0.020763200521469117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,31,0.023056000471115112
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,63,0.020768000185489653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,63,0.023073600232601167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,127,0.02295520007610321
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,127,0.021092799305915833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,255,0.03471199870109558
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,255,0.02877120077610016
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,511,0.04978080093860626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,511,0.04287199974060059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,1023,0.06311839818954468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,1023,0.08439679741859436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,127,0.03707999885082245
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,2047,0.09893280267715454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,2047,0.1458575963973999
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,4095,0.17368320226669312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,4095,0.26950559616088865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,1,0.012329600006341934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,1,0.013067199289798737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,float16,8191,0.5201407909393311
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,3,0.012188799679279327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,7,0.01223199963569641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,12,12,128,1,float16,fp8,8191,0.3223439931869507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,3,0.013283200562000275
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,7,0.013209599256515502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,15,0.013284799456596375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,15,0.012177599966526032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,31,0.012217599898576736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,31,0.013079999387264252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,63,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,63,0.013235199451446533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,127,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,127,0.013011200726032257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,255,0.014215999841690063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,511,0.017187200486660004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,255,0.015115199983119965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,511,0.018723200261592864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,1023,0.02072319984436035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,1023,0.02470560073852539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,2047,0.03912799954414368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,2047,0.03219679892063141
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,4095,0.0625551998615265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,4095,0.053934401273727416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,float16,8191,0.10836160182952881
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,12,12,128,1,float16,fp8,8191,0.08620960116386414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,1,0.05547680258750916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,1,0.06505600214004517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,3,0.055255997180938723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,3,0.0648576021194458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,7,0.05518879890441895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,7,0.06500800251960755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,15,0.0554144024848938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,31,0.05572479963302612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,15,0.06482880115509033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,31,0.06499199867248535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,63,0.0607200026512146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,63,0.06541759967803955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,127,0.0669327974319458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,127,0.07415040135383606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,255,0.10236159563064576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,255,0.09300959706306458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,511,0.15867199897766113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,511,0.13861440420150756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,1023,0.2836992025375366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,1023,0.21113920211791992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,2047,0.35608639717102053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,3,0.036950400471687316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,2047,0.5790112018585205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,1,0.10005919933319092
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,3,0.10006400346755981
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,1,0.16739360094070435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,3,0.12117919921875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,fp8,4095,0.648583984375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,7,0.09866080284118653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,12,12,128,1,float16,float16,4095,1.0220895767211915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,7,0.12115360498428344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,15,0.09783520102500916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,15,0.1192736029624939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,31,0.12147200107574463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,31,0.10118399858474732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,63,0.10641119480133057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,63,0.12528320550918579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,127,0.11377120018005371
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,127,0.14099680185317992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,255,0.16875360012054444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,255,0.1933168053627014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,511,0.30681281089782714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,511,0.25841119289398196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,1,0.1880352020263672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,1,0.22481439113616944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,float16,1023,0.5370687961578369
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,3,0.18742400407791138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,3,0.22761280536651612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,12,12,128,1,float16,fp8,1023,0.7478352069854737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,7,0.18358880281448364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,7,0.22704000473022462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,15,0.18818559646606445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,31,0.19441759586334229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,31,0.23431360721588135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,63,0.20053439140319823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,15,0.32492640018463137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,63,0.23692638874053956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,127,0.22019200325012206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,127,0.25511679649353025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,255,0.3451792001724243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,1,0.012878400087356568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,1,0.013935999572277069
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,255,0.32554240226745607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,3,0.012967999279499053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,7,0.012955200672149659
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,3,0.01342879980802536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,7,0.013780799508094788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,15,0.012591999769210816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,float16,511,0.600052785873413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,15,0.013793599605560303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,31,0.012724800407886505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,12,12,128,1,float16,fp8,511,0.5008944034576416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,31,0.01342879980802536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,63,0.013092799484729767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,63,0.013331200182437896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,255,0.01438080072402954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,255,0.01541599929332733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,127,0.012691199779510498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,127,0.013518400490283966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,511,0.0173007994890213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,511,0.019088000059127808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,1023,0.035420799255371095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,1023,0.030532801151275636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,2047,0.05275200009346008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,4095,0.08624320030212403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,2047,0.04572960138320923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,4095,0.0705407977104187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,fp8,8191,0.12212640047073364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,12,12,128,1,float16,float16,8191,0.15552639961242676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,1,0.35747199058532714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,1,0.4238368034362793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,3,0.35138399600982667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,7,0.35733919143676757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,3,0.634065580368042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,15,0.3677056074142456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,7,0.4241903781890869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,15,0.4348624229431152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,4095,0.3311055898666382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,31,0.7475152015686035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,63,0.3757791996002197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,31,0.45520482063293455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,63,0.458465576171875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,127,0.40808639526367185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,127,0.6460768222808838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,float16,255,0.653545618057251
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,12,12,128,1,float16,fp8,255,0.6175424098968506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,1,0.699180793762207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,1,0.8307120323181152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,7,0.7094655990600586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,7,0.8312687873840332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,3,0.8266703605651855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,3,1.1899423599243164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,15,0.7171008110046386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,31,0.7155392169952393
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,15,0.8994064331054688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,63,0.7270976066589355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,31,0.8841535568237304
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,float16,127,0.7713696002960205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,63,1.3077055931091308
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,1,0.0160303995013237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,1,0.016974399983882903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,7,0.017430399358272553
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,3,0.01738879978656769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,7,0.01746560037136078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,3,0.015798400342464446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,15,0.015707199275493623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,15,0.017441600561141968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,31,0.017343999445438386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,31,0.0169295996427536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,63,0.01568479984998703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,63,0.018822400271892546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,127,0.016161599755287172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,127,0.01695519983768463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,255,0.019280000030994414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,255,0.02312159985303879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,511,0.031411200761795044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,511,0.028142398595809935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,1023,0.05041760206222534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,1023,0.047131198644638064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,2047,0.08384000062942505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,12,12,128,1,float16,fp8,127,0.911184024810791
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,2047,0.06778560280799865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,1,0.023835200071334838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,4095,0.12248480319976807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,4095,0.15160000324249268
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,3,0.0438832014799118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,fp8,8191,0.2202928066253662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,12,12,128,1,float16,float16,8191,0.286516809463501
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,7,0.02436800003051758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,15,0.023870399594306944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,31,0.024089600145816802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,7,0.03675360083580017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,63,0.024534399807453155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,1,0.027107200026512145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,127,0.024820800125598907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,127,0.027102398872375488
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,255,0.0423119992017746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,511,0.06244159936904907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,511,0.05278559923171997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,1023,0.10596319437026977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,1023,0.07872160077095032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,2047,0.18877439498901366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,2047,0.1267680048942566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,15,0.027478399872779845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,4095,0.35432798862457277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,4095,0.2275712013244629
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,12,12,128,1,float16,fp8,15,0.03710399866104126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,8191,0.69027681350708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,8191,0.6053359985351563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,1,0.01808159947395325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,1,0.019351999461650848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,16383,0.8275247573852539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,3,0.01809120029211044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,7,0.01839040070772171
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,3,0.012201599776744843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,7,0.019675199687480927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,15,0.019896000623703003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,15,0.01170559972524643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,float16,16383,1.3565216064453125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,31,0.01855359971523285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,31,0.01234079971909523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,63,0.019865599274635316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,127,0.011737599968910217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,63,0.011724799871444702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,127,0.017372800409793852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,255,0.013326400518417358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,511,0.023083199560642243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,511,0.017905600368976593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,1023,0.022495999932289124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,1023,0.01690080016851425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,31,0.02778559923171997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,2047,0.023263999819755556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,2047,0.017283199727535246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,4095,0.018147200345993042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,4095,0.025915199518203737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,8191,0.021879999339580535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,8191,0.021902400255203246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,float16,16383,0.04217279851436615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,16383,0.028248000144958495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,1,0.011851199716329575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,1,0.014691199362277984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,3,0.012630400061607362
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,3,0.01154239997267723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,7,0.013846400380134582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,7,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,15,0.011519999802112579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,31,0.01162559986114502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,31,0.012427199631929398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,15,0.013531200587749481
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,63,0.011785600334405899
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,63,0.012283200025558471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,127,0.012303999811410903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,127,0.012665599584579468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,255,0.01348000019788742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,255,0.01518079936504364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,511,0.016607999801635742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,511,0.017961600422859193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,1023,0.016950400173664094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,1023,0.017192000150680543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,2047,0.01774719953536987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,2047,0.018692800402641298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,4095,0.021678400039672852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,4095,0.021167999505996703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,8191,0.037441599369049075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,8191,0.027422401309013366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,float16,16383,0.05862399935722351
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,8,128,1,float16,fp8,16383,0.04239839911460876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,1,0.01729599982500076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,1,0.01802240014076233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,3,0.016568000614643096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,3,0.01852319985628128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,7,0.0166143998503685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,7,0.01844000071287155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,15,0.016543999314308167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,15,0.01844480037689209
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,31,0.016711999475955964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,63,0.016659200191497803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,31,0.020390400290489198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,63,0.01808159947395325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,127,0.01910240054130554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,127,0.01806560009717941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,255,0.023614400625228883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,255,0.022256000339984892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,511,0.037668800354003905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,511,0.02984800040721893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,1023,0.06093440055847168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,1023,0.045956799387931825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,63,0.027432000637054442
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,2047,0.0684112012386322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,2047,0.10452480316162109
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,4095,0.18547040224075317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,4095,0.11939359903335571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,8191,0.3584847927093506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,8191,0.21891040802001954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,1,0.011856000125408172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,1,0.01268800050020218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,3,0.011761599779129028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,3,0.012608000636100769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,7,0.01175839975476265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,7,0.0126351997256279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,float16,16383,0.687656021118164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,15,0.011804799735546111
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,8,128,1,float16,fp8,16383,0.6735599994659424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,31,0.011857599765062333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,63,0.011804799735546111
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,15,0.012656000256538392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,31,0.03864159882068634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,63,0.012699200212955475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,255,0.01385599970817566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,127,0.01186399981379509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,127,0.03787040114402771
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,255,0.014628799259662628
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,511,0.01647839993238449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,1023,0.017550399899482726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,1023,0.01722400039434433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,511,0.05668479800224304
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,2047,0.020043200254440306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,2047,0.020824000239372253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,4095,0.055871999263763426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,4095,0.02717280089855194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,8191,0.05765600204467773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,8191,0.08159040212631226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,fp8,16383,0.06601600050926208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,8,128,1,float16,float16,16383,0.1003119945526123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,3,0.04047040045261383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,1,0.07980480194091796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,3,0.046342399716377256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,1,0.04600639939308167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,7,0.06601920127868652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,7,0.046532800793647765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,15,0.04014239907264709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,15,0.07658399939537049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,255,0.03611840009689331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,31,0.0404992014169693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,31,0.04608319997787476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,63,0.04668320119380951
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,63,0.06658400297164917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,127,0.0479312002658844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,127,0.04728319942951202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,255,0.07235519886016846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,255,0.10424480438232422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,511,0.09373279809951782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,511,0.11180319786071777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,1023,0.1970352053642273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,1023,0.1793887972831726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,2047,0.238753604888916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,2047,0.36089119911193845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,1,0.06963199973106385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,1,0.08273919820785522
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,4095,0.6913983821868896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,4095,0.4357312202453613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,3,0.06961920261383056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,7,0.07079359889030457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,3,0.08356800079345703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,7,0.08285920023918152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,15,0.07041919827461243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,31,0.06992959976196289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,15,0.08362560272216797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,31,0.08378720283508301
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,63,0.07545599937438965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,127,0.08604639768600464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,63,0.08565120100975036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,fp8,8191,0.8324943542480469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,127,0.09409279823303222
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,255,0.12961759567260742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,255,0.11809920072555542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,8,128,1,float16,float16,8191,2.0156719207763674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,511,0.1770751953125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,511,0.21066720485687257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,3,0.027590399980545043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,1,0.1302880048751831
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,1023,0.3681488037109375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,1,0.157532799243927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,3,0.12886879444122315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,3,0.15767840147018433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,2047,0.46102080345153806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,7,0.1285423994064331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,float16,2047,0.6988160133361816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,8,128,1,float16,fp8,1023,0.6088255882263184
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,7,0.15757919549942018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,15,0.1559391975402832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,15,0.12845439910888673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,31,0.1400704026222229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,63,0.1394592046737671
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,31,0.293067193031311
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,63,0.16548479795455934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,127,0.14666080474853516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,255,0.286572790145874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,127,0.17323999404907225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,255,0.23079519271850585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,511,0.40804319381713866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,1,0.011988800019025803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,511,0.3428704023361206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,3,0.011849600076675414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,1,0.012716799974441528
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,7,0.011840000003576278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,3,0.012753599882125854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,7,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,15,0.012148799747228623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,15,0.012641599774360657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,31,0.011912000179290772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,31,0.012827199697494508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,63,0.01194560006260872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,63,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,127,0.01218400001525879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,127,0.012680000066757202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,float16,1023,0.8539487838745117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,255,0.01356479972600937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,255,0.014798399806022645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,511,0.016867199540138246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,8,128,1,float16,fp8,1023,0.545689582824707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,1023,0.019939200580120088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,511,0.026873600482940675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,1023,0.02088479995727539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,2047,0.03619680106639862
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,2047,0.02778880000114441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,4095,0.05796639919281006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,4095,0.06182559728622437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,8191,0.1006767988204956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,8191,0.06664320230484008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,float16,16383,0.1849951982498169
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,8,128,1,float16,fp8,16383,0.1155743956565857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,3,0.24086239337921142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,1,0.36042559146881104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,1,0.2933216094970703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,7,0.23974719047546386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,3,0.29592161178588866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,7,0.29072959423065187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,15,0.29422080516815186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,15,0.3420703887939453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,31,0.253769588470459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,31,0.3142656087875366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,63,0.25774080753326417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,8,128,1,float16,fp8,255,0.014398400485515595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,63,0.31490559577941896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,127,0.34320640563964844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,127,0.32068800926208496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,255,0.40958237648010254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,255,0.41895198822021484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,3,0.46203198432922366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,1,0.6254608154296875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,fp8,511,0.6597856044769287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,8,128,1,float16,float16,511,0.8000016212463379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,1,0.5599103927612304
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,7,0.46831841468811036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,3,0.5691472053527832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,15,0.5781184196472168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,7,0.5586431980133056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,31,0.4930384159088135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,15,0.5851856231689453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,63,0.4930751800537109
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,31,0.6007872104644776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,127,0.5259151935577393
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,63,0.598083209991455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,1,0.011924800276756287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,7,0.011977600306272507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,127,0.6101920127868652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,1,0.012964800000190735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,3,0.01265919953584671
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,3,0.012203200161457062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,7,0.012945599853992462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,15,0.012705600261688233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,31,0.011896000057458878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,15,0.011948800086975098
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,31,0.01279360055923462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,63,0.012003199756145477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,float16,255,0.7804656028747559
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,63,0.012676799297332763
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,127,0.012091200053691863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,127,0.012583999335765839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,255,0.013795199990272521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,255,0.015007999539375306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,511,0.018279999494552612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,511,0.017080000042915343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,1023,0.03598879873752594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,1023,0.02584959864616394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,2047,0.05720319747924805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,2047,0.04033760130405426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,8,128,1,float16,fp8,255,1.1732975959777832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,4095,0.06525760293006896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,4095,0.09945759773254395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,1,0.017059199512004852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,8191,0.11471519470214844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,8191,0.18343839645385743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,3,0.017047999799251555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,7,0.016840000450611115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,float16,16383,0.3515935897827148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,8,128,1,float16,fp8,16383,0.3719167947769165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,15,0.016947199404239655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,31,0.026807999610900878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,63,0.0170864000916481
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,8,128,1,float16,fp8,7,0.027408000826835633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,127,0.01740960031747818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,1,0.01855040043592453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,255,0.020787200331687926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,255,0.022416000068187714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,511,0.03685120046138764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,1023,0.06061440110206604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,1023,0.046419200301170346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,2047,0.10376640558242797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,2047,0.0703440010547638
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,4095,0.18610880374908448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,4095,0.1195296049118042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,8191,0.3526992082595825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,15,0.01879359930753708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,8191,0.2182624101638794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,16383,0.4119872093200684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,16383,0.6888912200927735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,32767,0.7988783836364746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,float16,32767,1.357759952545166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,1,0.012294399738311767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,1,0.011403200030326844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,3,0.011582399904727935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,3,0.01226079985499382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,7,0.011535999923944473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,7,0.012252800166606903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,15,0.012352000176906585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,15,0.011606399714946748
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,31,0.011587200313806533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,31,0.012689599394798278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,63,0.011627200245857238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,63,0.012652799487113953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,127,0.011494400352239609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,127,0.012705600261688233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,255,0.012988799810409546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,255,0.014692799746990204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,511,0.01594240069389343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,511,0.01886560022830963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,1023,0.016363200545310975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,1023,0.018385599553585052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,2047,0.01671680063009262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,2047,0.01828480064868927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,4095,0.017030400037765504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,4095,0.01844319999217987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,8191,0.019569599628448488
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,8191,0.019233599305152893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,16383,0.02300640046596527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,16383,0.023001599311828613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,float16,32767,0.037796801328659056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,4,128,1,float16,fp8,32767,0.03136000037193298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,1,0.011566399782896041
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,1,0.012297599762678146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,3,0.011559999734163284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,3,0.012388800084590913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,7,0.011553599685430526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,7,0.01239359974861145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,31,0.01903519928455353
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,15,0.01175519973039627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,15,0.012449599802494049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,31,0.012385600060224534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,63,0.011692799627780914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,31,0.011769600212574005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,63,0.012350399792194367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,127,0.011670400202274323
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,127,0.01239520013332367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,255,0.013286399841308593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,255,0.01446239948272705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,511,0.016232000291347505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,511,0.018054400384426118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,1023,0.017948800325393678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,1023,0.017107200622558594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,2047,0.017339199781417847
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,4095,0.018907199800014495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,2047,0.018007999658584593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,4095,0.01844480037689209
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,8191,0.022337600588798523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,8191,0.022566400468349457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,16383,0.03821440041065216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,16383,0.028505599498748778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,float16,32767,0.06001120209693909
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,4,128,1,float16,fp8,32767,0.04463520050048828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,1,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,1,0.013652800023555756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,3,0.01266079992055893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,3,0.013697600364685059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,7,0.012651200592517852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,7,0.013510400056838989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,15,0.012689599394798278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,15,0.01412159949541092
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,31,0.012715199589729309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,31,0.013775999844074249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,63,0.012796799838542938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,63,0.013923199474811554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,127,0.012807999551296235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,127,0.013767999410629273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,255,0.014535999298095703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,255,0.01576479971408844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,511,0.018089599907398224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,511,0.01966560035943985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,1023,0.026785600185394286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,1023,0.03628320097923279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,2047,0.04110879898071289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,2047,0.058475202322006224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,4095,0.10059839487075806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,127,0.018943999707698823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,4095,0.06696159839630127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,8191,0.11676160097122193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,8191,0.18403680324554444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,16383,0.35545599460601807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,16383,0.21344640254974365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,1,0.011627200245857238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,1,0.012479999661445617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,3,0.011655999720096588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,3,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,float16,32767,0.689347219467163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,7,0.011777599900960922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,15,0.01175839975476265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,4,128,1,float16,fp8,32767,0.6717951774597168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,7,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,31,0.012452799826860428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,15,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,63,0.011734399944543839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,31,0.013488000631332398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,63,0.012516799569129943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,127,0.011633600294589996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,127,0.014270399510860444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,255,0.013739199936389923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,255,0.01440960019826889
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,511,0.0179967999458313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,511,0.017420800030231477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,1023,0.0171424001455307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,1023,0.01804479956626892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,2047,0.020070399343967437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,2047,0.018385599553585052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,4095,0.022382399439811705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,4095,0.02354400008916855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,8191,0.028118398785591126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,8191,0.03808479905128479
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,16383,0.06069440245628357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,16383,0.04269599914550781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,float16,32767,0.10269279479980468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,4,128,1,float16,fp8,32767,0.0712015986442566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,1,0.026046401262283324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,1,0.029374399781227113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,3,0.02900159955024719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,3,0.026679998636245726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,7,0.028998398780822755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,7,0.026199999451637267
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,15,0.02680160105228424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,15,0.028961598873138428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,31,0.02643040120601654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,31,0.029918399453163148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,63,0.025931200385093688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,63,0.02964800000190735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,127,0.029193601012229918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,127,0.026972800493240356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,255,0.04392800033092499
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,255,0.03752799928188324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,511,0.06427040100097656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,511,0.055068802833557126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,1023,0.10786720514297485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,1023,0.08015040159225464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,2047,0.19079359769821166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,2047,0.12796640396118164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,3,0.018555200099945067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,4095,0.22891199588775635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,4095,0.35410079956054685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,1,0.04282079935073853
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,8191,0.42596960067749023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,1,0.04903199970722198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,8191,0.6883423805236817
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,3,0.04910399913787842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,3,0.04305759966373444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,7,0.04339359998703003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,7,0.04885280132293701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,15,0.049099200963974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,15,0.04325760006904602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,31,0.042875200510025024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,31,0.049239999055862425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,63,0.04387679994106293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,63,0.04934720098972321
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,127,0.04943200051784515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,float16,16383,1.3542096138000488
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,4,128,1,float16,fp8,16383,1.9351951599121093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,127,0.04991680085659027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,255,0.07258399724960327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,255,0.07133600115776062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,511,0.09688320159912109
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,511,0.13681119680404663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,1023,0.19832320213317872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,1023,0.1478320002555847
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,2047,0.36041600704193116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,2047,0.28225278854370117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,1,0.07692000269889832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,1,0.115830397605896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,3,0.0770143985748291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,fp8,4095,0.4419600009918213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,4,128,1,float16,float16,4095,0.6911312103271484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,3,0.11580159664154052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,7,0.07687199711799622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,7,0.08965759873390197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,31,0.07692800164222717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,15,0.10898079872131347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,15,0.07681599855422974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,31,0.08867040276527405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,63,0.08061280250549316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,63,0.10597120523452759
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,127,0.08644480109214783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,127,0.09698240160942077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,255,0.12557920217514038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,511,0.1849120020866394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,255,0.13277920484542846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,511,0.2139375925064087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,511,0.030456000566482545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,1,0.012099199742078782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,1023,0.36974880695343015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,1023,0.2845344066619873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,1,0.012777599692344665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,3,0.012195199728012085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,3,0.012625600397586822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,7,0.012251199781894683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,7,0.012942400574684144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,15,0.011913599818944931
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,31,0.012116800248622894
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,15,0.013009600341320038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,31,0.01273919939994812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,63,0.011937599629163742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,63,0.012931199371814727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,fp8,2047,0.47260799407958987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,127,0.012316799908876418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,127,0.012984000146389008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,255,0.01388320028781891
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,511,0.016732800006866454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,511,0.01841440051794052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,1023,0.019023999571800232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,255,0.014587199687957764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,1023,0.018755200505256652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,4,128,1,float16,float16,2047,1.1294320106506348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,2047,0.021587200462818146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,4095,0.038022398948669434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,2047,0.022537599503993987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,8191,0.06214399933815003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,4095,0.029212799668312073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,16383,0.10230720043182373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,16383,0.06819679737091064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,8191,0.043166399002075195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,1,0.1379631996154785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,fp8,32767,0.11839359998703003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,3,0.13847839832305908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,4,128,1,float16,float16,32767,0.18868160247802734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,1,0.16281119585037232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,3,0.16346880197525024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,7,0.1379871964454651
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,15,0.1383344054222107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,7,0.16127359867095947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,15,0.16176480054855347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,31,0.14220479726791382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,31,0.16414239406585693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,63,0.14372800588607787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,63,0.17194559574127197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,127,0.14933279752731324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,127,0.1742751955986023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,255,0.22289760112762452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,255,0.22995519638061523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,511,0.3490272045135498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,511,0.39700319766998293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,1,0.26063520908355714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,1,0.3088848114013672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,3,0.25940799713134766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,7,0.2602799892425537
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,float16,1023,0.708193588256836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,3,0.3060415983200073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,7,0.30903680324554444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,15,0.3086767911911011
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,4,128,1,float16,fp8,1023,1.3698384284973144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,31,0.2631903886795044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,15,0.26008000373840334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,31,0.3220223903656006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,7,0.018771199882030486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,63,0.2661360025405884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,127,0.2754096031188965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,127,0.33000481128692627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,1,0.012116800248622894
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,255,0.407155179977417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,63,0.5154767990112304
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,1,0.012913599610328674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,255,0.4311552047729492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,3,0.012214399874210358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,7,0.01202080026268959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,3,0.018432000279426576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,7,0.01284639984369278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,15,0.012051200121641159
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,15,0.012985600531101227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,31,0.01733279973268509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,float16,511,0.7701024055480957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,4,128,1,float16,fp8,511,0.6654143810272217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,63,0.012049599736928939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,31,0.01273919939994812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,255,0.013846400380134582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,63,0.015143999457359314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,127,0.012140800058841706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,511,0.017998400330543517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,255,0.014724799990653991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,511,0.016782400012016297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,2047,0.03776159882545471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,127,0.012628799676895142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,1023,0.021748800575733186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,4095,0.04230400025844574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,4095,0.05963039994239807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,2047,0.028476798534393312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,1023,0.022041599452495574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,8191,0.10234240293502808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,8191,0.06721280217170715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,16383,0.1177135944366455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,1,0.01308799982070923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,16383,0.18572160005569457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,3,0.013148799538612366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,7,0.013391999900341034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,fp8,32767,0.22383520603179932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,4,128,1,float16,float16,32767,0.355348801612854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,15,0.013457599282264709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,31,0.013289600610733032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,63,0.013278399407863618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,4,128,1,float16,fp8,63,0.01886080056428909
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,127,0.013355199992656708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,255,0.015595200657844543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,1,0.014148800075054169
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,511,0.01828320026397705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,511,0.019806399941444397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,1023,0.037436801195144656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,1023,0.027700799703598022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,2047,0.05857920050621033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,2047,0.04177600145339966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,4095,0.10161919593811035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,4095,0.06658719778060913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,8191,0.18561760187149048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,8191,0.11791679859161378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,7,0.013991999626159667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,16383,0.35537919998168943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,16383,0.21389119625091552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,32767,0.6913072109222412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,32767,0.4077568054199219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,65535,0.8017472267150879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,float16,65535,1.3735759735107422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,1,0.011548800021409988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,1,0.012427199631929398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,255,0.016335999965667723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,3,0.011670400202274323
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,3,0.012249600142240524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,7,0.011547199636697768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,7,0.012276799976825714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,15,0.011495999991893768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,31,0.011564800143241882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,31,0.012167999893426895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,15,0.012412799894809723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,63,0.01154400035738945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,127,0.011443199962377549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,63,0.01239679977297783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,127,0.01236959993839264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,255,0.01297599971294403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,255,0.014470399916172027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,511,0.016103999316692354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,511,0.01804639995098114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,1023,0.017067199945449828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,1023,0.017924800515174866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,2047,0.01685280054807663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,2047,0.018379199504852294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,4095,0.017051200568675994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,4095,0.018033599853515624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,8191,0.019225600361824035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,8191,0.01988479942083359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,16383,0.020553599298000335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,16383,0.020880000293254854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,32767,0.024721600115299225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,32767,0.024636800587177276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,float16,65535,0.039971199631690976
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,1,0.011316800117492675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,2,128,1,float16,fp8,65535,0.03263840079307556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,1,0.012100800126791
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,3,0.01183520033955574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,3,0.012095999717712403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,7,0.011665599793195725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,7,0.011987199634313583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,15,0.011785600334405899
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,15,0.012078399956226348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,31,0.011817599833011627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,31,0.012163200229406358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,63,0.01180960014462471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,63,0.012153600156307221
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,127,0.011713600158691407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,127,0.012211199849843979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,255,0.013412800431251527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,255,0.01406240016222
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,511,0.016457599401474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,511,0.017844800651073457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,1023,0.017448000609874725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,1023,0.017790399491786957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,2047,0.017375999689102174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,2047,0.01786559969186783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,4095,0.017617599666118623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,4095,0.018052799999713896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,8191,0.020975999534130096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,8191,0.01974399983882904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,16383,0.024318400025367736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,16383,0.023651200532913207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,32767,0.03924480080604553
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,32767,0.030300799012184142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,float16,65535,0.06155520081520081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,2,128,1,float16,fp8,65535,0.04593439996242523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,1,0.012691199779510498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,63,0.014103999733924866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,1,0.013366399705410004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,3,0.012681600451469422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,3,0.013396799564361572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,7,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,7,0.013601599633693695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,15,0.012580800056457519
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,15,0.01321599930524826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,31,0.012729600071907043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,31,0.013417600095272065
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,63,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,63,0.013612799346446991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,127,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,127,0.013352000713348388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,255,0.014595200121402741
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,255,0.015156799554824829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,511,0.017343999445438386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,511,0.019407999515533448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,1023,0.022260800004005432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,1023,0.022867199778556824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,2047,0.039320001006126405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,2047,0.028856000304222106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,4095,0.060241597890853885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,4095,0.043747198581695554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,8191,0.07064480185508729
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,8191,0.10535199642181396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,16383,0.18939679861068726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,16383,0.12171200513839722
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,32767,0.35889599323272703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,32767,0.22320160865783692
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,1,0.011606399714946748
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,3,0.011454399675130844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,1,0.012582400441169738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,3,0.01223199963569641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,fp8,65535,0.41467838287353515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,7,0.011750400066375732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,7,0.01223519966006279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,2,128,1,float16,float16,65535,0.6972767829895019
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,15,0.012520000338554382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,31,0.01149279996752739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,15,0.0115167997777462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,63,0.011795199662446975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,127,0.011535999923944473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,63,0.01228640004992485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,31,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,127,0.012624000012874604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,255,0.013203200697898865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,255,0.014263999462127686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,511,0.018248000741004945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,511,0.016361600160598753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,1023,0.016846400499343873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,1023,0.018011200428009033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,2047,0.017387199401855468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,2047,0.018428799510002137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,4095,0.01860480010509491
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,4095,0.018651199340820313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,8191,0.024439999461174013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,8191,0.023982399702072145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,16383,0.030796799063682555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,16383,0.03930400013923645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,32767,0.06176959872245789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,32767,0.0463375985622406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,1,0.018889600038528444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,float16,65535,0.10478399991989136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,2,128,1,float16,fp8,65535,0.07290400266647339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,1,0.02054080069065094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,3,0.018603199720382692
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,3,0.020000000298023225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,7,0.01886879950761795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,7,0.020137600600719452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,15,0.018436799943447112
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,15,0.02022559940814972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,31,0.018537600338459016
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,31,0.01987680047750473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,63,0.01860480010509491
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,63,0.02009759992361069
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,127,0.018382400274276733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,127,0.020244799554347992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,255,0.022601599991321563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,255,0.023924799263477327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,127,0.014059199392795563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,511,0.03866240084171295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,511,0.032148799300193785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,1023,0.06178879737854004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,1023,0.04746240079402923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,2047,0.10512959957122803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,2047,0.07218239903450012
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,4095,0.18749599456787108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,4095,0.12163679599761963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,8191,0.22109920978546144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,8191,0.3560480117797852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,16383,0.41869440078735354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,16383,0.6887375831604003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,1,0.02815040051937103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,3,0.02815519869327545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,1,0.03065760135650635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,3,0.031112000346183777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,7,0.02784320116043091
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,7,0.031070399284362792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,15,0.02770400047302246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,15,0.031062400341033934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,31,0.027646398544311522
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,31,0.031134399771690368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,63,0.0278656005859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,63,0.031276801228523256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,fp8,32767,0.8057760238647461
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,127,0.028064000606536865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,127,0.031401601433753965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,2,128,1,float16,float16,32767,1.3606816291809083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,255,0.038790398836135866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,255,0.050044798851013185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,511,0.06634560227394104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,511,0.05699679851531982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,1023,0.08206719756126404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,1023,0.1112671971321106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,3,0.01419519931077957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,2047,0.19352320432662964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,2047,0.1292896032333374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,1,0.048184001445770265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,4095,0.23120479583740233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,4095,0.35909759998321533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,1,0.054176002740859985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,7,0.048100799322128296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,3,0.04896160066127777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,3,0.05454239845275879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,7,0.05399680137634277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,15,0.048974400758743285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,fp8,8191,0.4268640041351318
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,15,0.054739201068878175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,31,0.04824320077896118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,31,0.05485759973526001
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,2,128,1,float16,float16,8191,0.6886127948760986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,63,0.05493760108947754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,63,0.04843359887599945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,127,0.05607039928436279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,127,0.05569599866867066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,255,0.08265920281410218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,255,0.07206879854202271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,511,0.10126080513000488
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,511,0.12030080556869507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,1023,0.20634241104125978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,1023,0.15308480262756347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,1,0.011987199634313583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,1,0.012617599964141846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,3,0.012011200189590454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,3,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,2047,0.36562719345092776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,2047,0.24879679679870606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,7,0.01188960000872612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,7,0.012571200728416443
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,31,0.012012799829244613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,15,0.011793600022792816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,31,0.012583999335765839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,15,0.012969599664211273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,63,0.01202400028705597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,63,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,127,0.011959999799728394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,fp8,4095,0.44728641510009765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,127,0.012947200238704682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,255,0.013659200072288514
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,255,0.014345599710941315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,511,0.016646400094032288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,2,128,1,float16,float16,4095,0.6965472221374511
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,1023,0.01759839951992035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,1023,0.018592000007629395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,2047,0.018880000710487364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,511,0.018129600584506987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,2047,0.018313600122928618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,4095,0.022563199698925018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,4095,0.0225615993142128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,8191,0.030299198627471925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,8191,0.040092799067497256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,16383,0.04463199973106384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,16383,0.06121439933776855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,32767,0.07028319835662841
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,32767,0.10389440059661866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,float16,65535,0.18852319717407226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,1,0.08174399733543396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,3,0.09371200203895569
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,2,128,1,float16,fp8,65535,0.12190719842910766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,1,0.09356480240821838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,7,0.08328480124473572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,3,0.08307679891586303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,7,0.09395840167999267
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,15,0.08321599960327149
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,15,0.09388319849967956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,31,0.08369280099868774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,31,0.0951088011264801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,63,0.08617759943008423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,63,0.09729120135307312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,127,0.10249760150909423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,127,0.09161919951438904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,255,0.13768160343170166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,255,0.12746880054473878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,511,0.21390559673309326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,511,0.19165439605712892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,1023,0.2901407957077026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,1023,0.37264161109924315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,1,0.14952000379562377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,3,0.17260960340499878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,1,0.17210719585418702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,3,0.14685920476913453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,float16,2047,0.7058080196380615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,2,128,1,float16,fp8,2047,0.47894558906555174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,7,0.1494431972503662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,7,0.17236800193786622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,15,0.14677120447158815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,31,0.17336000204086305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,15,0.17320159673690796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,31,0.15174560546875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,63,0.15676480531692505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,63,0.18097440004348755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,127,0.1638592004776001
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,127,0.187009596824646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,255,0.23848159313201905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,255,0.23860640525817872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,15,0.01465280055999756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,1,0.01180799975991249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,1,0.01284479945898056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,3,0.011772800236940384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,3,0.012859199941158295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,511,0.39540801048278806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,7,0.01180960014462471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,7,0.012849600613117218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,511,0.3636415958404541
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,15,0.01281599998474121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,31,0.012099199742078782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,15,0.011966399848461151
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,63,0.012377600371837615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,63,0.013142399489879608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,31,0.01263359934091568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,127,0.012359999865293504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,127,0.01327040046453476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,255,0.013662399351596832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,255,0.01493919938802719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,511,0.016763199865818024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,511,0.01831520050764084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,1023,0.019068799912929535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,fp8,1023,0.5619344234466552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,1023,0.018825599551200868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,2,128,1,float16,float16,1023,0.7158256053924561
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,2047,0.020857599377632142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,2047,0.022678400576114654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,4095,0.03854880034923554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,4095,0.028969600796699524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,8191,0.06155040264129639
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,16383,0.07049279808998107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,8191,0.04740639925003052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,16383,0.10486880540847779
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,32767,0.18869119882583618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,32767,0.12276639938354492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,1,0.013412800431251527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,7,0.013425600528717042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,3,0.01363999992609024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,float16,65535,0.3581279993057251
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,15,0.013264000415802002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,2,128,1,float16,fp8,65535,0.2548255920410156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,31,0.013222399353981017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,63,0.01358720064163208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,127,0.013235199451446533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,1,0.013891200721263885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,255,0.014923200011253357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,2,128,1,float16,fp8,31,0.014444799721240997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,255,0.01581439971923828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,511,0.018030400574207305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,511,0.019606399536132812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,1023,0.02269600033760071
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,1023,0.02346560060977936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,2047,0.039556801319122314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,2047,0.029627200961112977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,4095,0.06090720295906067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,4095,0.04362399876117706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,8191,0.1056447982788086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,8191,0.07052479982376099
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,16383,0.19388959407806397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,16383,0.1255071997642517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,63,0.014019200205802917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,32767,0.36157119274139404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,32767,0.22598559856414796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,65535,0.4201456069946289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,65535,0.7013040065765381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,1,0.011452800035476685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,1,0.012185599654912949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,3,0.011393599957227708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,float16,131071,1.3741791725158692
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,3,0.012055999785661697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,131071,0.8045344352722168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,7,0.012035199999809265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,7,0.011537600308656693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,15,0.012172800302505494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,15,0.011425600200891495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,31,0.01157120019197464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,31,0.012078399956226348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,63,0.01149279996752739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,127,0.011479999870061874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,63,0.012323199957609176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,255,0.013123199343681335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,127,0.012095999717712403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,255,0.014404800534248353
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,511,0.016043199598789214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,511,0.017951999604701997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,1023,0.016884799301624297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,1023,0.017903999984264375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,2047,0.018028800189495087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,2047,0.016715200245380403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,4095,0.016996799409389494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,4095,0.018219199776649476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,8191,0.018859200179576874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,8191,0.01990559995174408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,16383,0.022526399791240694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,16383,0.023080000281333925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,32767,0.023664000630378722
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,32767,0.024223999679088594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,65535,0.027449598908424376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,65535,0.02717440128326416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,1,0.011358399689197541
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,fp8,131071,0.035123199224472046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,1,1,128,1,float16,float16,131071,0.04617120027542114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,1,0.012118399888277055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,3,0.011540800333023071
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,3,0.012033600360155106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,7,0.011390399932861329
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,7,0.012377600371837615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,15,0.011327999830245971
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,15,0.012108799815177918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,31,0.01146399974822998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,31,0.012011200189590454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,63,0.011407999694347382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,63,0.01223360002040863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,127,0.011385600268840789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,127,0.01207199990749359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,255,0.012910400331020356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,255,0.013971200585365296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,511,0.015881599485874177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,511,0.018001599609851836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,1023,0.017827199399471284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,1023,0.016705599427223206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,2047,0.01669279932975769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,2047,0.017987200617790224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,4095,0.017020800709724428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,4095,0.017903999984264375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,127,0.014327999949455262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,8191,0.01889439970254898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,8191,0.019849599897861482
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,16383,0.023902399837970732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,16383,0.023336000740528107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,32767,0.026868799328804018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,32767,0.026907199621200563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,65535,0.042535999417304994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,65535,0.034211200475692746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,1,0.01226079985499382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,float16,131071,0.06482880115509033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,1,1,128,1,float16,fp8,131071,0.04910880029201507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,1,0.013531200587749481
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,3,0.01236959993839264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,3,0.013203200697898865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,7,0.013411200046539307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,7,0.01223199963569641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,15,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,15,0.013060800731182098
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,31,0.012723200023174286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,31,0.01300320029258728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,63,0.013388800621032714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,63,0.012404800206422806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,127,0.012838399410247803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,127,0.013495999574661254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,255,0.014126400649547576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,255,0.015544000267982482
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,511,0.017150400578975676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,511,0.01876160055398941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,1023,0.019702400267124175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,1023,0.01929119974374771
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,2047,0.02115360051393509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,2047,0.0234047994017601
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,4095,0.03926079869270325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,4095,0.028622400760650635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,8191,0.06215360164642334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,8191,0.044809600710868834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,16383,0.07469919919967652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,16383,0.108024001121521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,32767,0.1932479977607727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,32767,0.12733919620513917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,65535,0.3616976022720337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,65535,0.22537600994110107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,1,0.011446399986743927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,1,0.012086399644613267
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,float16,131071,0.6996575832366944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,1,1,128,1,float16,fp8,131071,0.4219359874725342
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,3,0.011433599889278412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,7,0.0122079998254776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,3,0.01220960021018982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,7,0.011396799981594086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,15,0.01143999993801117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,3,0.014257599413394929
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,31,0.011470399796962738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,15,0.012204799801111221
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,31,0.012268800288438797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,63,0.01173119992017746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,63,0.012299200147390365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,127,0.0115167997777462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,127,0.01223199963569641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,255,0.01329759955406189
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,255,0.014230400323867798
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,511,0.01804479956626892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,511,0.016091200709342956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,1023,0.017153599858283998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,1023,0.01798879951238632
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,2047,0.01796800047159195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,2047,0.01693760007619858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,4095,0.017416000366210938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,4095,0.01817599982023239
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,8191,0.020472000539302825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,8191,0.02008640021085739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,16383,0.027375999093055724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,16383,0.027559998631477355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,32767,0.03322399854660034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,32767,0.043012800812721255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,65535,0.06506400108337403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,65535,0.050310397148132326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,1,0.014727999269962311
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,float16,131071,0.10905280113220214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,1,1,128,1,float16,fp8,131071,0.07661759853363037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,1,0.01605439931154251
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,3,0.014654399454593658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,7,0.014561599493026734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,3,0.01526239961385727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,7,0.015943999588489532
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,15,0.014819200336933135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,15,0.015300799906253815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,31,0.01462559998035431
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,31,0.015593600273132325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,63,0.014678399264812469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,63,0.015460799634456634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,127,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,127,0.016008000075817107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,255,0.01650719940662384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,255,0.01720000058412552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,511,0.02057439982891083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,511,0.021862399578094483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,1023,0.028948798775672913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,1023,0.03822399973869324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,2047,0.059164798259735106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,2047,0.04302079975605011
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,4095,0.06808000206947326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,4095,0.10220320224761963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,8191,0.187336003780365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,8191,0.11999679803848266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,16383,0.21767680644989013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,16383,0.35587520599365235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,31,0.014347200095653535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,32767,0.4181951999664307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,1,0.02101760059595108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,1,0.022385600209236144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,32767,0.6952032089233399
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,3,0.022720000147819518
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,7,0.020849600434303284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,3,0.020923200249671935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,15,0.020977599918842314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,15,0.02260800004005432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,7,0.022724799811840057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,31,0.022708800435066224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,31,0.021004800498485566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,63,0.021006399393081666
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,127,0.020953600108623505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,127,0.02260479927062988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,63,0.022758400440216063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,255,0.024145600199699403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,255,0.02627040147781372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,511,0.04162560105323791
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,511,0.034332799911499026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,fp8,65535,0.8133184432983398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,1023,0.06539520025253295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,1023,0.05011680126190186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,2047,0.10712319612503052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,1,1,128,1,float16,float16,65535,1.3700960159301758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,2047,0.0770799994468689
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,4095,0.12437599897384644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,4095,0.19089599847793579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,1,0.03450399935245514
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,8191,0.36090240478515623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,1,0.03750079870223999
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,3,0.03746879994869232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,8191,0.23952319622039794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,3,0.04748480021953583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,7,0.03452160060405731
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,15,0.0373744010925293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,15,0.059724801778793336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,fp8,16383,0.4208367824554443
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,7,0.03773120045661926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,31,0.034353598952293396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,31,0.06500959992408753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,1,1,128,1,float16,float16,16383,0.6932655811309815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,63,0.03777279853820801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,127,0.03478559851646423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,63,0.03406400084495544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,127,0.06520000100135803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,255,0.04626879990100861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,511,0.061921602487564086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,255,0.053668802976608275
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,511,0.07261120080947876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,1023,0.13774880170822143
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,1023,0.09151520133018494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,2047,0.1370479941368103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,2047,0.20003199577331543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,1,0.011715199798345566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,1,0.012521600723266602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,4095,0.36392960548400877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,3,0.011767999827861786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,4095,0.2908576011657715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,7,0.011776000261306763
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,fp8,8191,0.4367551803588867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,3,0.012291199713945388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,15,0.011878400295972823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,7,0.013790400326251983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,31,0.011604800075292587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,31,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,63,0.013041600584983826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,15,0.012534399330615998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,63,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,1,1,128,1,float16,float16,8191,0.6983104228973389
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,127,0.011819200217723846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,127,0.012375999987125397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,255,0.013398399949073792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,255,0.01610880047082901
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,511,0.018199999630451203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,1023,0.0171984001994133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,1023,0.018248000741004945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,2047,0.019395199418067933
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,511,0.016436800360679626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,4095,0.019144000113010408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,2047,0.01835840046405792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,4095,0.018665599822998046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,8191,0.027060800790786745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,8191,0.024111999571323393
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,16383,0.0432096004486084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,16383,0.033548799157142636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,32767,0.06473919749259949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,32767,0.05574719905853272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,65535,0.10904480218887329
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,65535,0.07756800055503846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,1,0.06244800090789795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,1,0.06112160086631775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,fp8,131071,0.12856320142745972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,3,0.055238401889801024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,1,1,128,1,float16,float16,131071,0.19615360498428344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,3,0.06781119704246522
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,7,0.05543839931488037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,15,0.05489919781684875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,15,0.06090880036354065
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,7,0.060043197870254514
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,31,0.06127840280532837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,31,0.061355197429656984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,63,0.06016799807548523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,63,0.05596479773521423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,127,0.06176000237464905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,127,0.0694320023059845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,255,0.08464480042457581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,255,0.07760800123214721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,511,0.12409600019454955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,511,0.11014720201492309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,1023,0.16353119611740113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,1023,0.2145632028579712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,7,0.014009599387645722
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,2047,0.2557471990585327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,2047,0.37035839557647704
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,1,0.10533119440078735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,1,0.09562879800796509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,3,0.1051408052444458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,7,0.10727519989013672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,3,0.09429600238800048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,7,0.09295840263366699
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,15,0.09374560117721557
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,15,0.1074944019317627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,31,0.09464160203933716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,fp8,4095,0.4547296047210693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,31,0.10531680583953858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,1,1,128,1,float16,float16,4095,0.7021215915679931
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,63,0.10981600284576416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,63,0.0996783971786499
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,127,0.10478399991989136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,255,0.14372960329055787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,127,0.11464159488677979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,255,0.14213919639587402
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,511,0.21440320014953612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,511,0.201475191116333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,1,0.011819200217723846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,1,0.01276959925889969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,3,0.011796800047159195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,1023,0.3782416105270386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,3,0.012692800164222718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,1023,0.30622398853302
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,7,0.011724799871444702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,15,0.011937599629163742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,31,0.011870399862527848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,7,0.012902399897575379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,31,0.012804800271987915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,15,0.012931199371814727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,63,0.011820799857378005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,63,0.012884800136089326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,127,0.011699199676513672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,127,0.012838399410247803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,fp8,2047,0.49035038948059084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,255,0.013753600418567657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,255,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,511,0.016519999504089354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,1,1,128,1,float16,float16,2047,0.7118288040161133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,511,0.01847199946641922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,1023,0.01871519982814789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,2047,0.019196799397468566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,1023,0.01728159934282303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,2047,0.01881919950246811
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,4095,0.02253440022468567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,4095,0.02239679992198944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,8191,0.04027679860591889
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,8191,0.030131199955940248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,16383,0.06414399743080139
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,16383,0.048388800024986266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,32767,0.1077455997467041
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,65535,0.1921231985092163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,32767,0.07424160242080688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,1,0.013799999654293061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,1,0.014752000570297241
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,65535,0.12579200267791749
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,3,0.013944000005722046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,3,0.014558400213718414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,7,0.013969600200653076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,7,0.01454080045223236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,float16,131071,0.3617680072784424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,15,0.013827200233936309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,1,1,128,1,float16,fp8,131071,0.22267520427703857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,31,0.01388159990310669
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,15,0.014670400321483612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,31,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,63,0.01451359987258911
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,63,0.013950400054454803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,127,0.014531199634075165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,127,0.014254400134086609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,255,0.015625600516796113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,255,0.016359999775886536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,511,0.01879040002822876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,511,0.020094400644302367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,1023,0.03358879983425141
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,1,0.013433599472045898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,1023,0.026804798841476442
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,1,0.014300799369812012
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,float16,2047,0.04683519899845123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,1,128,1,float16,fp8,2047,0.0363103985786438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,3,0.01422560065984726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,3,0.013337600231170654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,7,0.013238400220870972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,7,0.014342400431632995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,15,0.013495999574661254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,15,0.014263999462127686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,31,0.013465599715709686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,31,0.014310400187969207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,63,0.01419840008020401
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,63,0.013364799320697784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,127,0.013944000005722046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,255,0.015591999888420105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,127,0.014268800616264343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,255,0.016201600432395935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,511,0.02008160054683685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,511,0.019817599654197694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,1023,0.0401775985956192
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,1023,0.029180800914764403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,1,0.017056000232696534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,fp8,2047,0.04338079988956452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,2,128,1,float16,float16,2047,0.06118080019950867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,1,0.018857599794864656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,3,0.01730400025844574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,3,0.01855359971523285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,15,0.01727679967880249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,7,0.018913599848747253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,7,0.01730400025844574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,15,0.018777599930763243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,31,0.017075200378894807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,31,0.018796800076961516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,63,0.017455999553203583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,63,0.018760000169277192
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,127,0.017521600425243377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,127,0.01849759966135025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,255,0.022275200486183165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,511,0.03968159854412079
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,255,0.02282399982213974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,511,0.031115201115608216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,1023,0.06459680199623108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,1023,0.04856640100479126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,1,0.024710400402545928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,float16,2047,0.10708800554275513
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,1,0.027272000908851624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,4,128,1,float16,fp8,2047,0.07177919745445252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,1,1,128,1,float16,fp8,15,0.01414719969034195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,3,0.024694399535655977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,3,0.02762719988822937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,7,0.02797439992427826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,7,0.02465119957923889
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,15,0.024352000653743745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,31,0.024801599979400634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,31,0.02778719961643219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,15,0.02771199941635132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,63,0.02486560046672821
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,63,0.02800639867782593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,127,0.027220800518989563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,127,0.02736639976501465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,255,0.044761601090431216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,255,0.03672159910202026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,511,0.06490560173988343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,511,0.05289440155029297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,1023,0.1102336049079895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,1,0.011830399930477142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,1023,0.079995197057724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,1,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,3,0.011737599968910217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,7,0.01175519973039627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,fp8,2047,0.1275231957435608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,128,8,128,1,float16,float16,2047,0.19284319877624512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,15,0.01191679984331131
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,3,0.012622399628162384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,15,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,7,0.012559999525547028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,31,0.011747200042009354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,31,0.012729600071907043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,63,0.012670400738716125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,127,0.011929599940776825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,127,0.012612800300121307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,255,0.014531199634075165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,255,0.013465599715709686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,63,0.011700800061225891
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,511,0.016763199865818024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,511,0.01855839937925339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,1023,0.017052799463272095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,1023,0.017899200320243835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,float16,2047,0.017131200432777403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,1,128,1,float16,fp8,2047,0.018084800243377684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,1,0.011505600064992905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,1,0.012567999958992004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,3,0.011566399782896041
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,3,0.01371839940547943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,7,0.01156959980726242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,7,0.012392000108957291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,15,0.011633600294589996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,15,0.012675200402736665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,31,0.01180960014462471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,31,0.012537600100040435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,63,0.012768000364303589
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,63,0.012383999675512314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,127,0.01170239970088005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,127,0.01231039986014366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,255,0.01327040046453476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,255,0.014596800506114959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,511,0.016312000155448914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,511,0.020399999618530274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,1023,0.01722400039434433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,1023,0.016169600188732147
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,float16,2047,0.01655520051717758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,1,0.01183359995484352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,2,128,1,float16,fp8,2047,0.01749120056629181
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,1,0.012326399981975555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,3,0.012798400223255157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,3,0.012345600128173827
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,7,0.011574400216341018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,7,0.012427199631929398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,15,0.011707200109958649
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,15,0.012254399806261062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,31,0.01149279996752739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,63,0.011767999827861786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,31,0.013571199774742127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,63,0.012292800098657608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,127,0.011617600172758102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,127,0.012200000137090683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,255,0.013198399543762207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,255,0.014452800154685974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,511,0.017107200622558594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,511,0.018254399299621582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,1023,0.015987199544906617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,1023,0.01693120002746582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,float16,2047,0.016174399852752687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,4,128,1,float16,fp8,2047,0.017395199835300447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,1,0.012300799787044524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,1,0.012107200175523757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,3,0.01162559986114502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,3,0.01226079985499382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,7,0.011433599889278412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,15,0.011649599671363831
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,7,0.012403199821710587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,15,0.013055999577045441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,31,0.01154239997267723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,31,0.012292800098657608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,63,0.011470399796962738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,63,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,127,0.011660800129175187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,127,0.013027200102806091
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,255,0.013238400220870972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,255,0.014219200611114502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,1023,0.01600160002708435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,511,0.016238400340080263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,511,0.01808159947395325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,float16,2047,0.01655520051717758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,1023,0.017931200563907623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,128,8,128,1,float16,fp8,2047,0.017550399899482726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,1,0.011795199662446975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,1,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,3,0.011772800236940384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,3,0.012628799676895142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,7,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,7,0.012652799487113953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,15,0.011846400052309036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,15,0.012435200065374375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,31,0.011791999638080596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,31,0.012718400359153748
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,63,0.012100800126791
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,63,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,127,0.012027200311422348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,127,0.012379200011491776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,255,0.013521599769592284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,255,0.014555199444293976
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,511,0.016764800250530242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,511,0.01851679980754852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,1023,0.017046399414539337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,1023,0.017892800271511078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,float16,2047,0.01724960058927536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,1,0.01180960014462471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,1,0.012303999811410903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,1,128,1,float16,fp8,2047,0.01835999935865402
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,3,0.011742399632930755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,3,0.012263999879360199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,7,0.011736000329256058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,7,0.012670400738716125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,15,0.011724799871444702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,15,0.012612800300121307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,31,0.01199679970741272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,63,0.01183520033955574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,31,0.012297599762678146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,63,0.012755200266838074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,127,0.011814399808645248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,127,0.012580800056457519
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,255,0.01332319974899292
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,255,0.014494399726390838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,511,0.016155199706554414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,511,0.01818079948425293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,1023,0.01754239946603775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,1023,0.016484799981117248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,float16,2047,0.016708800196647645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,2,128,1,float16,fp8,2047,0.01793439984321594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,1,0.011420799791812897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,1,0.012371200323104858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,3,0.011736000329256058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,3,0.012529599666595458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,7,0.011737599968910217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,7,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,15,0.011451199650764465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,15,0.012382400035858155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,31,0.01186240017414093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,63,0.011720000207424164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,31,0.012588800489902496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,63,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,127,0.011556799709796905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,127,0.012532800436019897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,255,0.013279999792575835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,255,0.0144896000623703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,511,0.016451199352741242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,511,0.01812639981508255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,1023,0.015992000699043274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,1023,0.0172447994351387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,float16,2047,0.01650879979133606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,4,128,1,float16,fp8,2047,0.01759999990463257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,1,0.011774399876594543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,1,0.012409599870443344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,3,0.011907199770212174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,3,0.012624000012874604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,7,0.011748799681663513
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,7,0.012591999769210816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,15,0.011646399646997452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,15,0.012600000202655792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,31,0.011803200095891952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,31,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,63,0.011879999935626984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,63,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,127,0.01188800036907196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,127,0.012390399724245072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,255,0.013704000413417817
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,255,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,511,0.016264000535011293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,511,0.018236799538135527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,1023,0.016462400555610657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,1023,0.017350399494171144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,fp8,2047,0.01764480024576187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,128,8,128,1,float16,float16,2047,0.018408000469207764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,1,0.013107199966907502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,1,0.013876800239086152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,3,0.013041600584983826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,3,0.013716800510883332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,7,0.01340160071849823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,7,0.013763199746608733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,15,0.013126400113105775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,15,0.013689599931240082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,31,0.012878400087356568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,31,0.0141744002699852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,63,0.01327199935913086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,63,0.013713599741458892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,127,0.013736000657081604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,127,0.01324480026960373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,255,0.015454399585723876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,255,0.015723200142383577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,511,0.017790399491786957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,511,0.01908479928970337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,1023,0.021385599672794343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,1023,0.0217071995139122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,float16,2047,0.02781760096549988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,1,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,1,128,1,float16,fp8,2047,0.025153601169586183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,1,0.013391999900341034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,3,0.013142399489879608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,3,0.013396799564361572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,7,0.012547199428081513
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,7,0.013600000739097595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,15,0.012625600397586822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,15,0.013953599333763122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,31,0.013359999656677246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,31,0.0126351997256279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,63,0.012835200130939483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,63,0.013436800241470337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,127,0.013422399759292603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,127,0.013372799754142762
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,255,0.015214399993419647
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,255,0.014731200039386749
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,511,0.019356800615787505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,511,0.017627200484275816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,1023,0.022865599393844603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,1023,0.02252960056066513
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,1,0.012548799812793731
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,fp8,2047,0.030430400371551515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,2,128,1,float16,float16,2047,0.04132480025291443
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,1,0.013449600338935852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,3,0.013619199395179749
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,3,0.012627199292182922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,7,0.013859200477600097
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,7,0.0126351997256279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,15,0.012636800110340119
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,15,0.0135343998670578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,31,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,31,0.013822400569915771
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,63,0.012806400656700134
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,63,0.013764800131320953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,127,0.012863999605178833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,127,0.01358560025691986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,255,0.014460800588130951
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,255,0.01576319932937622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,511,0.018515199422836304
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,511,0.019038400053977965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,1023,0.03740319907665253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,1023,0.027420800924301148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,float16,2047,0.05923519730567932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,1,0.01656319946050644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,4,128,1,float16,fp8,2047,0.041171199083328246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,1,0.01780640035867691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,3,0.017892800271511078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,3,0.016897599399089813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,7,0.01645279973745346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,7,0.017907199263572694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,15,0.017766399681568144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,15,0.01679839938879013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,31,0.016599999368190767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,31,0.018195199966430663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,63,0.016603200137615202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,63,0.017931200563907623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,127,0.016652800142765045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,127,0.018030400574207305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,255,0.021328000724315642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,255,0.02181279957294464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,511,0.02933279871940613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,511,0.037567999958992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,1023,0.06196799874305725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,1023,0.04657599925994873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,1,0.012060800194740295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,1,0.012566399574279786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,float16,2047,0.10318399667739868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,3,0.011972799897193909
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,128,8,128,1,float16,fp8,2047,0.07051039934158325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,3,0.012649600207805634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,7,0.011929599940776825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,7,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,15,0.011897599697113037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,31,0.011977600306272507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,15,0.01257600039243698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,31,0.012699200212955475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,63,0.011828800290822982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,63,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,127,0.012065599858760833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,127,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,255,0.01358879953622818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,255,0.014395199716091156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,511,0.01659200042486191
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,511,0.01849599927663803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,1023,0.01700959950685501
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,1023,0.017998400330543517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,float16,2047,0.017524799704551695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,1,128,1,float16,fp8,2047,0.01850239932537079
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,1,0.011707200109958649
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,1,0.01252640038728714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,3,0.011579199880361556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,3,0.012303999811410903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,7,0.011803200095891952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,7,0.012587200105190276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,15,0.011840000003576278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,15,0.012267199903726577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,31,0.011711999773979187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,31,0.012569600343704223
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,63,0.011852800101041793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,127,0.01167680025100708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,63,0.012385600060224534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,127,0.012296000123023986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,255,0.0134320005774498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,255,0.014396800100803376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,511,0.016331200301647187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,511,0.01810079962015152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,1023,0.016334399580955505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,1023,0.017689600586891174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,float16,2047,0.01719679981470108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,2,128,1,float16,fp8,2047,0.017764799296855927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,1,0.011791999638080596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,1,0.012403199821710587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,3,0.011772800236940384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,3,0.012604799866676331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,7,0.011785600334405899
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,7,0.01242239996790886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,15,0.011580800265073776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,15,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,31,0.011760000139474869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,31,0.012529599666595458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,63,0.011727999895811081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,63,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,127,0.011761599779129028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,127,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,255,0.013758400082588195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,255,0.014510400593280792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,511,0.01629280000925064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,511,0.01797440052032471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,1023,0.016595199704170227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,1023,0.017228800058364867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,float16,2047,0.018278400599956512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,1,0.012639999389648438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,1,0.011790399998426437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,4,128,1,float16,fp8,2047,0.017825600504875184
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,3,0.011984000355005265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,3,0.012825599312782288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,7,0.011736000329256058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,7,0.012671999633312225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,15,0.01180960014462471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,15,0.012638400495052337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,31,0.01202239990234375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,31,0.012563200294971466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,63,0.012060800194740295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,63,0.01271039992570877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,127,0.012089599668979645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,127,0.012566399574279786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,255,0.014163200557231904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,255,0.014608000218868256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,511,0.01677920073270798
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,511,0.018305599689483643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,1023,0.0182096004486084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,1023,0.017497600615024568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,float16,2047,0.0204815998673439
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,1,0.015251199901103973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,128,8,128,1,float16,fp8,2047,0.021241599321365358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,1,0.01655520051717758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,3,0.015889599919319153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,3,0.016062399744987486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,7,0.01584320068359375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,7,0.016596800088882445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,15,0.015883199870586395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,15,0.016118399798870087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,31,0.01571359932422638
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,31,0.01655679941177368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,63,0.015868799388408662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,63,0.016020800173282623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,127,0.01757120043039322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,127,0.016791999340057373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,255,0.019543999433517457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,255,0.018161599338054658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,511,0.02375040054321289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,511,0.022227199375629426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,1,0.018481600284576415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,float16,1023,0.042931199073791504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,1,0.02062239944934845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,3,0.018775999546051025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,1,128,1,float16,fp8,1023,0.03195360004901886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,3,0.020555199682712556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,7,0.01849759966135025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,7,0.02059199959039688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,15,0.019016000628471374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,15,0.020046399533748628
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,31,0.01890240013599396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,31,0.020584000647068022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,63,0.018620799481868743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,63,0.020664000511169435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,127,0.020640000700950623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,127,0.019046400487422944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,255,0.02614080011844635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,255,0.02433760017156601
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,511,0.0423552006483078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,511,0.03330399990081787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,float16,1023,0.06792479753494263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,1,0.026072001457214354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,1,0.029707199335098265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,2,128,1,float16,fp8,1023,0.05023840069770813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,3,0.02637760043144226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,7,0.026100799441337585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,3,0.02874560058116913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,7,0.029531198740005492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,15,0.029543998837471008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,15,0.026443201303482055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,31,0.028880000114440918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,31,0.026038399338722228
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,63,0.0264847993850708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,63,0.02954080104827881
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,127,0.03128640055656433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,127,0.029211199283599852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,255,0.04739840030670166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,255,0.03919839859008789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,511,0.06898720264434814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,511,0.05573760271072388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,1,0.04107680022716522
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,float16,1023,0.11414560079574584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,1,0.04699519872665405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,3,0.04103040099143982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,4,128,1,float16,fp8,1023,0.08379200100898743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,3,0.04702079892158508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,7,0.040191999077796935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,7,0.04699999988079071
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,15,0.04100480079650879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,31,0.04108319878578186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,15,0.04680800139904022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,31,0.046587198972702026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,63,0.04493759870529175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,63,0.04715520143508911
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,127,0.05018720030784607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,127,0.051318401098251344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,255,0.07636160254478455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,255,0.06753600239753724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,1,0.021593600511550903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,511,0.11738239526748658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,511,0.09620479941368103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,1,0.023262399435043334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,fp8,1023,0.14637600183486937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,128,8,128,1,float16,float16,1023,0.20398879051208496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,3,0.022257600724697114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,3,0.023809599876403808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,7,0.02210720032453537
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,7,0.023145599663257597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,15,0.021377600729465485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,15,0.023915199935436247
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,31,0.02369759976863861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,63,0.023449599742889404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,31,0.02156960070133209
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,63,0.02268480062484741
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,127,0.023705600202083586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,127,0.023145599663257597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,1,0.028518399596214293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,fp8,255,0.027331200242042542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,1,128,1,float16,float16,255,0.035729598999023435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,1,0.03234719932079315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,3,0.03158720135688782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,3,0.028753599524497984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,7,0.028683200478553772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,7,0.03173440098762512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,15,0.02868959903717041
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,15,0.03239839971065521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,31,0.028603199124336242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,31,0.03189760148525238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,63,0.03174560070037842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,63,0.029817599058151244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,127,0.03804959952831268
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,127,0.03240320086479187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,float16,255,0.051235198974609375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,2,128,1,float16,fp8,255,0.04371359944343567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,1,0.042972800135612485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,1,0.049726399779319766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,3,0.04328320026397705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,3,0.04948799908161163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,7,0.04298079907894135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,7,0.04918720126152039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,15,0.04344640076160431
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,15,0.0495824009180069
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,31,0.04366079866886139
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,31,0.04933759868144989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,63,0.05015519857406616
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,63,0.050379198789596555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,127,0.05672320127487183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,127,0.057259202003479004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,float16,255,0.07580959796905518
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,1,0.07195839881896973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,4,128,1,float16,fp8,255,0.07027999758720398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,1,0.08505600094795226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,3,0.07195360064506531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,3,0.08428639769554139
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,15,0.07185919880867005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,7,0.07203680276870728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,7,0.08436639904975891
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,15,0.08493919968605042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,31,0.07605599761009216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,31,0.08422719836235046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,63,0.08069279789924622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,63,0.09162880182266235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,127,0.08929280042648316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,127,0.10024000406265259
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,float16,255,0.12941759824752808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,1,0.03711200058460236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,1,0.038987201452255246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,3,0.03834879994392395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,128,8,128,1,float16,fp8,255,0.12108319997787476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,7,0.03714720010757446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,3,0.03709119856357575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,7,0.03958399891853333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,15,0.0384768009185791
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,15,0.03865920007228851
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,31,0.03972319960594177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,31,0.03845599889755249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,63,0.04132800102233887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,63,0.04401600062847137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,float16,127,0.048609599471092224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,1,128,1,float16,fp8,127,0.04394879937171936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,1,0.05059840083122254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,1,0.05548160076141358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,3,0.051846402883529666
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,3,0.0553551971912384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,7,0.05111839771270752
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,7,0.05550079941749573
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,15,0.05184000134468079
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,15,0.05573760271072388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,31,0.056720000505447385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,31,0.056145602464675905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,63,0.05906559824943543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,63,0.060052800178527835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,float16,127,0.06603999733924866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,2,128,1,float16,fp8,127,0.06348639726638794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,1,0.08361759781837463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,1,0.08946560025215149
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,3,0.08029760122299194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,3,0.08999840021133423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,7,0.08204799890518188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,7,0.08996639847755432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,15,0.08569279909133912
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,15,0.08990560173988342
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,31,0.08598399758338929
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,31,0.09321439862251282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,63,0.08815680146217346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,63,0.09841279983520508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,float16,127,0.10023360252380371
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,4,128,1,float16,fp8,127,0.10235519409179687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,1,0.13789440393447877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,3,0.13854559659957885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,1,0.15756160020828247
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,7,0.13972320556640624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,3,0.15843839645385743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,7,0.17038400173187257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,15,0.1421712040901184
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,15,0.16021759510040284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,31,0.14418879747390748
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,63,0.14535679817199706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,1,0.012243200093507767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,31,0.16865600347518922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,1,0.01257600039243698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,63,0.17147840261459352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,float16,127,0.17493280172348022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,3,0.01279519945383072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,128,8,128,1,float16,fp8,127,0.18164800405502318
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,3,0.011998400092124939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,7,0.012001600116491318
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,7,0.01297439932823181
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,15,0.012889599800109864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,15,0.01446399986743927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,63,0.012062399834394454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,31,0.012587200105190276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,31,0.011987199634313583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,63,0.012796799838542938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,127,0.01226240023970604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,127,0.015334400534629821
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,255,0.013556799292564392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,255,0.014508800208568573
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,511,0.018505600094795228
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,511,0.016572800278663636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,1023,0.01725119948387146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,1023,0.021742400527000428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,float16,2047,0.018240000307559966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,1,0.011847999691963196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,1,128,1,float16,fp8,2047,0.018534399569034576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,1,0.012691199779510498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,3,0.011873599886894227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,3,0.015063999593257904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,7,0.011923199892044068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,7,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,15,0.012724800407886505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,15,0.01188800036907196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,31,0.012062399834394454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,31,0.014667199552059173
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,63,0.012619200348854064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,63,0.012011200189590454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,127,0.012028799951076507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,127,0.012535999715328216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,255,0.01379680037498474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,255,0.017100800573825837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,511,0.016638399660587312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,511,0.017975999414920805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,1023,0.017041599750518797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,1023,0.01758880019187927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,1,0.012217599898576736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,float16,2047,0.018910400569438934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,2,128,1,float16,fp8,2047,0.021190400421619415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,1,0.012503999471664428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,3,0.011900799721479416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,7,0.012449599802494049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,3,0.012636800110340119
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,15,0.012548799812793731
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,31,0.011820799857378005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,7,0.012761600315570831
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,15,0.013739199936389923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,31,0.012928000092506409
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,63,0.012144000083208085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,63,0.012540799379348756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,127,0.012321600317955017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,127,0.013084800541400909
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,255,0.01619359999895096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,255,0.014735999703407287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,511,0.016868799924850464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,511,0.01823360025882721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,1023,0.018078400194644927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,1023,0.018880000710487364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,1,0.012104000151157378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,1,0.01271200031042099
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,float16,2047,0.020937600731849672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,4,128,1,float16,fp8,2047,0.021329599618911742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,3,0.01202400028705597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,7,0.011961600184440613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,3,0.013675199449062347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,7,0.01279519945383072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,15,0.01207360029220581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,31,0.01202239990234375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,15,0.013036799430847169
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,31,0.012779200077056884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,63,0.012928000092506409
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,63,0.012782399356365205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,127,0.012374400347471236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,127,0.012969599664211273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,255,0.014120000600814819
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,255,0.014590400457382201
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,511,0.018195199966430663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,511,0.018294399976730345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,1023,0.020497600734233856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,1023,0.021057599782943727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,float16,2047,0.0375216007232666
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,128,8,128,1,float16,fp8,2047,0.027531200647354127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,1,0.06842719912528991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,1,0.07037760019302368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,3,0.06721439957618713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,3,0.07044479846954346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,7,0.0681168019771576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,7,0.0703216016292572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,15,0.0694159984588623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,15,0.07064480185508729
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,31,0.06989279985427857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,float16,63,0.07315679788589477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,31,0.07123039960861206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,1,128,1,float16,fp8,63,0.07283520102500915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,1,0.09571840167045594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,1,0.10400960445404053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,3,0.09416319727897644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,3,0.10416320562362671
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,7,0.09425280094146729
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,7,0.10688960552215576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,15,0.10470240116119385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,15,0.09475679993629456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,31,0.09669600129127502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,31,0.105948805809021
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,float16,63,0.10723520517349243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,2,128,1,float16,fp8,63,0.10828479528427123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,1,0.149126398563385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,1,0.17293440103530883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,3,0.14903839826583862
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,3,0.19201279878616334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,7,0.1493615984916687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,7,0.17271679639816284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,15,0.15036319494247435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,15,0.17357439994812013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,31,0.18252480030059814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,31,0.17529759407043458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,float16,63,0.15557119846343995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,4,128,1,float16,fp8,63,0.1794592022895813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,1,0.26491360664367675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,3,0.2644063949584961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,1,0.4011104106903076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,3,0.3128191947937012
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,7,0.26507680416107177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,7,0.31308159828186033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,15,0.26590719223022463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,31,0.2697200059890747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,15,0.43213438987731934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,31,0.31669440269470217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,float16,63,0.27227039337158204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,128,8,128,1,float16,fp8,63,0.31982879638671874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,1,0.11619679927825928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,3,0.11722719669342041
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,1,0.16109280586242675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,3,0.12502239942550658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,7,0.11713919639587403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,7,0.1248703956604004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,15,0.1254480004310608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,15,0.11974560022354126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,1,128,1,float16,float16,31,0.14725600481033324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,1,128,1,float16,fp8,31,0.12611199617385865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,1,0.16892640590667723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,3,0.16851199865341188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,1,0.19124159812927247
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,3,0.1905311942100525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,7,0.17059839963912965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,15,0.17029279470443726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,7,0.25174560546875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,15,0.19194560050964354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,2,128,1,float16,float16,31,0.17258720397949218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,2,128,1,float16,fp8,31,0.19408479928970337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,1,0.2781152009963989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,1,0.32883200645446775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,3,0.2785856008529663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,7,0.2788271903991699
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,7,0.3271359920501709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,3,0.41930079460144043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,15,0.28088319301605225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,15,0.32818241119384767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,4,128,1,float16,float16,31,0.28527040481567384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,4,128,1,float16,fp8,31,0.3308527946472168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,1,0.508132791519165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,1,0.6024672031402588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,3,0.603982400894165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,7,0.5075695991516114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,3,0.6764976024627686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,1,0.011913599818944931
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,15,0.5092016220092773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,1,0.018083199858665466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,7,0.6077055931091309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,3,0.011958400160074234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,3,0.018003199994564057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,7,0.01202080026268959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,7,0.012940800189971924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,15,0.012089599668979645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,15,0.6029200077056884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,15,0.01810240000486374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,31,0.012144000083208085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,31,0.012963199615478515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,63,0.01196800023317337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,127,0.016944000124931337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,63,0.012675200402736665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,8,128,1,float16,float16,31,0.5103568077087403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,127,0.012566399574279786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,255,0.014339199662208557
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,511,0.023817600309848787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,255,0.014763200283050537
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,511,0.018369600176811218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,128,8,128,1,float16,fp8,31,0.6082367897033691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,1023,0.01786080002784729
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,1023,0.018534399569034576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,1,0.012878400087356568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,1,0.011881600320339202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,fp8,2047,0.026846399903297423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,1,128,1,float16,float16,2047,0.020598399639129638
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,3,0.012596799433231354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,7,0.011740799993276596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,3,0.01189119964838028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,7,0.012758399546146392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,15,0.014441600441932679
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,15,0.012902399897575379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,31,0.011750400066375732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,31,0.012652799487113953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,63,0.011825600266456604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,63,0.012875199317932129
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,127,0.014579200744628906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,127,0.012884800136089326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,255,0.01366720050573349
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,255,0.01438400000333786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,511,0.01683039963245392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,511,0.018171200156211854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,1023,0.022115199267864226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,1023,0.018428799510002137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,1,0.011856000125408172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,1,0.012680000066757202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,float16,2047,0.021462400257587434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,2,128,1,float16,fp8,2047,0.021844799816608428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,3,0.014294399321079254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,7,0.011838400363922119
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,3,0.013011200726032257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,7,0.012720000743865967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,15,0.015273599326610566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,31,0.012583999335765839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,15,0.011811199784278869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,63,0.012120000272989272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,31,0.011956799775362015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,63,0.012899200618267059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,127,0.013089600205421447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,127,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,255,0.013792000710964203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,255,0.014769600331783294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,511,0.016574400663375854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,511,0.01828480064868927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,1023,0.022252799570560457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,1023,0.020820799469947814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,float16,2047,0.03860479891300202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,1,0.011876799911260606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,1,0.012950399518013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,4,128,1,float16,fp8,2047,0.028406399488449096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,3,0.012695999443531036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,3,0.012988799810409546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,7,0.01263200044631958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,15,0.012110400199890136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,7,0.011840000003576278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,31,0.011878400295972823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,15,0.012824000418186187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,63,0.011919999867677689
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,31,0.013939200341701508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,63,0.012915199995040894
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,127,0.01207519993185997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,127,0.012702399492263794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,255,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,255,0.014068800210952758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,511,0.018457600474357606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,511,0.0178399994969368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,1023,0.036164799332618715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,1023,0.0255840003490448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,1,0.013950400054454803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,float16,2047,0.05754719972610474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,1,0.015113599598407745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,128,8,128,1,float16,fp8,2047,0.04116320013999939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,3,0.01343040019273758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,7,0.013483199477195739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,3,0.01440960019826889
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,7,0.014188799262046813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,15,0.014983999729156493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,15,0.013633599877357483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,31,0.013835200667381286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,31,0.014268800616264343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,63,0.01371839940547943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,255,0.015166400372982025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,63,0.014377599954605103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,127,0.013652800023555756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,127,0.015081599354743958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,255,0.016383999586105348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,511,0.01863040030002594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,511,0.01955360025167465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,1023,0.028113600611686707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,1,0.013529600203037262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,1,0.014075200259685516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,1023,0.02547999918460846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,float16,2047,0.044614401459693906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,3,0.013489599525928497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,1,128,1,float16,fp8,2047,0.03311200141906738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,7,0.013652800023555756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,3,0.014206400513648987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,7,0.014008000493049622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,15,0.013387200236320496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,15,0.014153599739074707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,31,0.013836799561977387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,63,0.013598400354385375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,63,0.01406240016222
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,127,0.01419519931077957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,31,0.014084799587726593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,127,0.013795199990272521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,255,0.015167999267578124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,255,0.016206400096416475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,511,0.01997919976711273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,511,0.01956000030040741
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,1023,0.028488001227378844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,1023,0.038812801241874695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,1,0.01725279986858368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,fp8,2047,0.04193440079689026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,1,0.018353599309921264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,2,128,1,float16,float16,2047,0.0599727988243103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,3,0.01698720008134842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,3,0.018299199640750885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,7,0.017126399278640746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,15,0.01701759994029999
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,7,0.019124799966812135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,31,0.017033599317073822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,15,0.01815840005874634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,31,0.01836320012807846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,63,0.017192000150680543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,63,0.018427200615406036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,127,0.017873600125312805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,127,0.01857440024614334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,255,0.021532799303531646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,511,0.038703998923301695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,255,0.022392000257968902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,511,0.030878400802612303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,1023,0.04771679937839508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,1023,0.06361600160598754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,1,0.024560000002384185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,1,0.02693600058555603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,fp8,2047,0.0705680012702942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,3,0.024307200312614442
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,4,128,1,float16,float16,2047,0.10586400032043457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,7,0.024532799422740937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,3,0.02775999903678894
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,7,0.02688319981098175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,15,0.02727360129356384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,15,0.024568000435829164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,31,0.02714880108833313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,31,0.02412479966878891
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,63,0.02542079985141754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,63,0.02698560059070587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,127,0.026555201411247252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,127,0.027534401416778563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,255,0.03570080101490021
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,255,0.0439983993768692
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,511,0.05261920094490051
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,511,0.0641759991645813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,1023,0.07839040160179138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,1023,0.10969280004501343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,1,0.011311999708414077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,1,0.012670400738716125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,fp8,2047,0.1253664016723633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,3,0.011694400012493134
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,7,0.01136159971356392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,96,8,128,1,float16,float16,2047,0.1918879985809326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,3,0.012276799976825714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,7,0.012680000066757202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,15,0.012289600074291229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,31,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,63,0.011715199798345566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,63,0.012887999415397644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,31,0.011499200016260147
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,15,0.011406400054693223
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,127,0.011566399782896041
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,127,0.012387199699878693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,255,0.013191999495029449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,511,0.016150400042533875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,255,0.014289599657058717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,511,0.01801760047674179
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,1023,0.017123199999332428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,1023,0.017564800381660462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,float16,2047,0.01663679927587509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,1,128,1,float16,fp8,2047,0.017871999740600587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,1,0.011449600011110306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,3,0.011952000111341477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,1,0.012145599722862244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,3,0.012307199835777282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,7,0.011400000005960465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,7,0.012244799733161926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,15,0.011396799981594086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,15,0.012822400033473968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,31,0.011488000303506852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,31,0.012392000108957291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,63,0.011297599971294403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,63,0.012355200201272964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,127,0.011513599753379821
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,127,0.012727999687194824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,255,0.013123199343681335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,255,0.014297600090503692
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,511,0.016033600270748138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,511,0.017947199940681457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,1023,0.015892800688743592
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,1023,0.017500799894332886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,float16,2047,0.016208000481128693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,1,0.011336000263690948
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,2,128,1,float16,fp8,2047,0.017529599368572235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,1,0.012193600088357926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,3,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,3,0.011396799981594086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,7,0.012195199728012085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,7,0.01133280023932457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,15,0.011556799709796905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,15,0.012464000284671784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,31,0.011500799655914306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,31,0.012188799679279327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,63,0.011420799791812897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,63,0.012305600196123123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,127,0.01191840022802353
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,127,0.012377600371837615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,255,0.01305440068244934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,255,0.014003199338912965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,511,0.018531200289726258
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,511,0.016331200301647187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,1023,0.015729600191116334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,1023,0.016806399822235106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,float16,2047,0.016014400124549865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,1,0.01162559986114502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,4,128,1,float16,fp8,2047,0.01764799952507019
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,3,0.011505600064992905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,1,0.012336000055074691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,3,0.012163200229406358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,7,0.011665599793195725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,15,0.011500799655914306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,7,0.012628799676895142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,15,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,31,0.01144160032272339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,31,0.012363199889659882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,63,0.01151840016245842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,63,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,127,0.011667200177907944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,127,0.012127999961376191
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,255,0.013225600123405457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,255,0.014190399646759033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,511,0.01788160055875778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,511,0.016513599455356597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,1023,0.015982399880886077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,1023,0.01703200042247772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,float16,2047,0.016422399878501893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,96,8,128,1,float16,fp8,2047,0.0173552006483078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,1,0.0118367999792099
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,1,0.01231359988451004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,3,0.011534400284290314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,3,0.012323199957609176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,7,0.011524800211191177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,7,0.012608000636100769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,15,0.011521600186824799
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,15,0.01228479966521263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,31,0.01170239970088005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,31,0.012265600264072418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,63,0.011939200013875962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,63,0.012212800234556198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,127,0.01162400022149086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,127,0.012206400185823441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,255,0.013273599743843078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,255,0.01459999978542328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,511,0.01623679995536804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,511,0.018108800053596497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,1023,0.01637440025806427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,1023,0.01763039976358414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,1,0.011591999977827071
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,float16,2047,0.01696320027112961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,1,0.01223680004477501
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,1,128,1,float16,fp8,2047,0.017684799432754517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,3,0.01146719977259636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,3,0.012316799908876418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,7,0.011681599915027619
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,7,0.012254399806261062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,15,0.012273599952459335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,15,0.01151840016245842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,31,0.011584000289440155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,31,0.012612800300121307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,63,0.011726400256156922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,63,0.012328000366687774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,127,0.011622399836778641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,127,0.012588800489902496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,255,0.013391999900341034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,255,0.014377599954605103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,511,0.01622239947319031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,511,0.017910400032997133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,1023,0.016307200491428375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,1023,0.0174687996506691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,float16,2047,0.01660960018634796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,1,0.011558400094509124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,2,128,1,float16,fp8,2047,0.017360000312328337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,1,0.012359999865293504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,3,0.011747200042009354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,3,0.012435200065374375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,7,0.012214399874210358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,7,0.011687999963760376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,15,0.011744000017642975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,15,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,31,0.011716800183057785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,31,0.012240000069141388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,63,0.011537600308656693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,63,0.012403199821710587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,127,0.011764799803495407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,127,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,255,0.013228799402713775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,255,0.014144000411033631
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,511,0.016371199488639833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,511,0.017953599989414214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,1023,0.015889599919319153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,1023,0.016899199783802034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,float16,2047,0.01687840074300766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,4,128,1,float16,fp8,2047,0.017392000555992125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,1,0.011671999841928482
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,1,0.012270399928092956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,3,0.01225920021533966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,3,0.011660800129175187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,7,0.01156959980726242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,7,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,15,0.012355200201272964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,15,0.011649599671363831
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,31,0.011648000031709672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,31,0.01231039986014366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,63,0.011788800358772278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,63,0.012383999675512314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,127,0.011736000329256058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,127,0.012211199849843979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,255,0.013708800077438354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,255,0.014291200041770934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,511,0.016340799629688263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,511,0.018033599853515624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,1023,0.01618880033493042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,1023,0.017073599994182585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,float16,2047,0.018137599527835845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,1,0.012796799838542938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,96,8,128,1,float16,fp8,2047,0.01746239960193634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,1,0.013387200236320496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,3,0.012824000418186187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,3,0.013260799646377563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,7,0.0130048006772995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,7,0.01372160017490387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,15,0.013812799751758576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,15,0.012665599584579468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,31,0.012726399302482604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,31,0.01366720050573349
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,63,0.013006399571895599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,63,0.013435199856758118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,127,0.013465599715709686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,127,0.013247999548912048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,255,0.015086400508880615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,255,0.01566080003976822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,511,0.0179967999458313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,511,0.019023999571800232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,1023,0.02072799950838089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,1023,0.019995200634002685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,1,0.01284639984369278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,1,0.013512000441551208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,float16,2047,0.025859200954437257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,1,128,1,float16,fp8,2047,0.02370239943265915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,3,0.012905600666999816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,3,0.013494400680065155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,7,0.012814399600028992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,7,0.013414399325847625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,31,0.012824000418186187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,31,0.013356800377368926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,15,0.0137472003698349
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,15,0.013065600395202636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,63,0.012748800218105316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,63,0.013396799564361572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,127,0.013227200508117676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,127,0.01337919980287552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,255,0.014748799800872802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,255,0.015731200575828552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,511,0.017836800217628478
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,511,0.019288000464439393
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,1023,0.022758400440216063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,1,0.01284639984369278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,2047,0.029238399863243104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,1,0.01361120045185089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,fp8,1023,0.022596800327301027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,3,0.013366399705410004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,2,128,1,float16,float16,2047,0.040566399693489075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,3,0.012814399600028992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,7,0.012992000579833985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,15,0.013020800054073333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,7,0.01369439959526062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,31,0.013683199882507324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,15,0.013505600392818451
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,31,0.012876799702644348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,63,0.01372320055961609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,63,0.012883199751377106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,127,0.013143999874591828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,255,0.014801600575447082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,255,0.015851199626922607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,127,0.013627199828624726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,511,0.018588800728321076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,511,0.019228799641132353
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,1023,0.026526400446891786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,1023,0.037529599666595456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,fp8,2047,0.04121119976043701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,4,128,1,float16,float16,2047,0.05905439853668213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,1,0.016966399550437928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,1,0.01798879951238632
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,3,0.016543999314308167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,15,0.016940799355506898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,3,0.01807519942522049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,7,0.016700799763202667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,31,0.016459199786186218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,7,0.017892800271511078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,15,0.01796640008687973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,31,0.01802240014076233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,63,0.01674560010433197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,63,0.017824000120162962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,127,0.01696320027112961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,255,0.021076799929142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,127,0.01794559955596924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,511,0.03687199950218201
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,255,0.02215680032968521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,511,0.030054399371147157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,1023,0.045998400449752806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,1023,0.06009759902954102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,1,0.011577600240707397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,fp8,2047,0.06973119974136352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,96,8,128,1,float16,float16,2047,0.10376640558242797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,1,0.012383999675512314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,3,0.011633600294589996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,3,0.012388800084590913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,7,0.012323199957609176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,7,0.011788800358772278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,15,0.012390399724245072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,15,0.011795199662446975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,31,0.011787199974060058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,31,0.012283200025558471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,63,0.01175519973039627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,63,0.01247519999742508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,127,0.011793600022792816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,127,0.012454400211572647
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,255,0.013260799646377563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,255,0.01430879980325699
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,511,0.016326400637626647
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,511,0.018379199504852294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,1023,0.01681919991970062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,float16,2047,0.017127999663352968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,1023,0.01748639941215515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,1,128,1,float16,fp8,2047,0.01801919937133789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,1,0.01173280030488968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,1,0.012438400089740754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,3,0.011718399822711945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,3,0.01226079985499382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,7,0.012436799705028534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,7,0.011603199690580369
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,15,0.011675199866294861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,15,0.012540799379348756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,31,0.011604800075292587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,31,0.012535999715328216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,63,0.011604800075292587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,63,0.012307199835777282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,127,0.01185920014977455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,127,0.012177599966526032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,255,0.014319999516010285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,255,0.013303999602794648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,511,0.016353599727153778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,511,0.01807200014591217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,1023,0.01624480038881302
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,float16,2047,0.016996799409389494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,1023,0.017390400171279907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,2,128,1,float16,fp8,2047,0.01762399971485138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,1,0.011828800290822982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,1,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,3,0.011844799667596818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,3,0.012611199915409089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,7,0.01165279969573021
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,7,0.012476799637079239
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,15,0.0117296002805233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,15,0.012556800246238708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,31,0.011787199974060058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,31,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,63,0.011856000125408172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,63,0.012521600723266602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,127,0.0118367999792099
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,127,0.012294399738311767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,255,0.014619199931621552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,255,0.013406400382518769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,511,0.016680000722408293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,511,0.018243199586868285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,1023,0.01629440039396286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,1023,0.017283199727535246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,float16,2047,0.018174399435520173
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,4,128,1,float16,fp8,2047,0.017737600207328796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,1,0.011977600306272507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,1,0.012755200266838074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,3,0.011923199892044068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,3,0.0126351997256279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,7,0.011980800330638886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,7,0.012902399897575379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,15,0.011913599818944931
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,15,0.012727999687194824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,31,0.011873599886894227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,31,0.012868799269199371
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,63,0.01202080026268959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,63,0.012748800218105316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,127,0.012603199481964112
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,127,0.012145599722862244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,255,0.013728000223636627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,511,0.01658719927072525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,255,0.014716799557209014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,511,0.018435199558734894
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,1023,0.017972800135612487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,1023,0.017369599640369417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,float16,2047,0.020534400641918183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,96,8,128,1,float16,fp8,2047,0.021211199462413788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,1,0.01483519971370697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,1,0.015520000457763672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,3,0.014767999947071075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,3,0.01616799980401993
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,7,0.014899200201034546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,7,0.015624000132083893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,15,0.014603200554847717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,15,0.016156800091266632
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,31,0.01483519971370697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,31,0.015479999780654907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,63,0.015025599300861359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,63,0.01584800034761429
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,127,0.01605439931154251
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,127,0.01564960032701492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,255,0.018147200345993042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,255,0.018030400574207305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,511,0.022603200376033784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,511,0.02112320065498352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,1,0.01844319999217987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,float16,1023,0.041547200083732604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,3,0.018488000333309173
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,1,128,1,float16,fp8,1023,0.03171519935131073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,1,0.020411199331283568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,3,0.019996799528598785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,7,0.018352000415325163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,7,0.020308800041675568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,31,0.01836639940738678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,15,0.020347200334072113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,15,0.018681600689888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,31,0.020049600303173064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,63,0.018671999871730804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,63,0.020054399967193604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,127,0.019169600307941438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,127,0.020652799308300017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,255,0.025987198948860167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,255,0.024299199879169463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,511,0.03272959887981415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,511,0.04217279851436615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,float16,1023,0.06669279932975769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,1,0.026017600297927858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,1,0.028961598873138428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,3,0.02625280022621155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,2,128,1,float16,fp8,1023,0.04953120052814484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,7,0.026107200980186464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,3,0.029323199391365053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,7,0.029207998514175416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,15,0.026183998584747313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,31,0.026150399446487428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,15,0.029071998596191407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,31,0.029310399293899538
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,63,0.026096001267433167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,63,0.02935839891433716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,127,0.029382398724555968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,127,0.02933279871940613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,255,0.038431999087333676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,255,0.046070399880409243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,511,0.06785119771957397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,511,0.05465279817581177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,1,0.04059039950370789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,1,0.046744000911712644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,3,0.040468800067901614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,float16,1023,0.11277120113372803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,4,128,1,float16,fp8,1023,0.08145920038223267
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,3,0.04640640020370483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,7,0.040740799903869626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,7,0.046489599347114566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,15,0.04051679968833923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,31,0.0468095988035202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,15,0.04672319889068603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,31,0.040345600247383116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,63,0.04316959977149963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,63,0.04678399860858917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,127,0.050483202934265135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,127,0.04925439953804016
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,255,0.07560639977455139
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,255,0.0669647991657257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,511,0.11651359796524048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,511,0.09610400199890137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,1,0.02272319942712784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,1,0.021712000668048858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,3,0.02181120067834854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,3,0.02295520007610321
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,7,0.02104160040616989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,7,0.023446400463581086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,fp8,1023,0.14516960382461547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,96,8,128,1,float16,float16,1023,0.2023263931274414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,15,0.021809600293636322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,15,0.022833600640296936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,31,0.02146880030632019
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,31,0.02353920042514801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,63,0.021140800416469575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,127,0.022921599447727203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,255,0.03430559933185577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,float16,127,0.022995199263095855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,63,0.02313919961452484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,1,128,1,float16,fp8,255,0.02751680016517639
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,1,0.028038400411605834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,1,0.03153280019760132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,3,0.028246399760246278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,3,0.03193280100822449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,7,0.028646400570869444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,7,0.03115679919719696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,15,0.028401601314544677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,15,0.03126559853553772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,31,0.028195199370384217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,31,0.03212159872055054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,63,0.028300800919532777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,63,0.031707200407981875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,127,0.0376336008310318
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,127,0.03166559934616089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,float16,255,0.04908480048179627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,2,128,1,float16,fp8,255,0.042401599884033206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,1,0.04343039989471435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,1,0.04879679977893829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,3,0.04307200014591217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,7,0.04258559942245484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,7,0.049481600522994995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,3,0.04955999851226807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,15,0.04268159866333008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,15,0.049236801266670224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,31,0.04259679913520813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,31,0.04970879852771759
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,63,0.04838719964027405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,63,0.04901120066642761
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,127,0.05565919876098633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,127,0.05585600137710571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,fp8,255,0.07039679884910584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,4,128,1,float16,float16,255,0.07621759772300721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,1,0.07185760140419006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,1,0.08367040157318115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,3,0.0715183973312378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,3,0.0836080014705658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,7,0.07162399888038636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,7,0.08376479744911194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,15,0.07199199795722962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,15,0.08387200236320495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,31,0.07404639720916747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,31,0.08376160264015198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,63,0.08074880242347718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,63,0.09011039733886719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,1,0.035043200850486754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,127,0.08447039723396302
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,127,0.09835519790649414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,float16,255,0.13012640476226806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,96,8,128,1,float16,fp8,255,0.12014240026473999
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,1,0.03840959966182709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,3,0.035313600301742555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,7,0.034904000163078305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,3,0.03806079924106598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,15,0.035183998942375186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,7,0.03781439960002899
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,15,0.03770720064640045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,31,0.035452800989151004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,63,0.04049760103225708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,31,0.03805919885635376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,63,0.038441601395607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,float16,127,0.04551520049571991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,1,128,1,float16,fp8,127,0.03924799859523773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,1,0.04907999932765961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,1,0.05478079915046692
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,3,0.04877919852733612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,3,0.055155199766159055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,7,0.04886400103569031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,7,0.05437440276145935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,15,0.05480639934539795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,15,0.04917280077934265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,31,0.05019999742507934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,63,0.05655360221862793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,31,0.05571680068969727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,63,0.05674239993095398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,float16,127,0.06299200057983398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,2,128,1,float16,fp8,127,0.0618511974811554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,1,0.07651360034942627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,3,0.07701759934425353
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,1,0.0890608012676239
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,3,0.08900799751281738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,7,0.07715680003166199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,7,0.08987680077552795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,15,0.0770799994468689
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,15,0.09038400053977966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,31,0.08395519852638245
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,31,0.09040960073471069
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,63,0.08519039750099182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,63,0.09763680100440979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,float16,127,0.09465439915657044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,1,0.13126399517059326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,3,0.13112959861755372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,1,0.15629760026931763
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,3,0.15736160278320313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,4,128,1,float16,fp8,127,0.10601919889450073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,7,0.1318384051322937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,7,0.15621440410614013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,15,0.13919520378112793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,15,0.1565392017364502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,31,0.1442415952682495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,31,0.17321759462356567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,63,0.14408960342407226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,63,0.16988799571990967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,float16,127,0.15346720218658447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,1,0.012705600261688233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,1,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,3,0.011812800168991089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,96,8,128,1,float16,fp8,127,0.18646880388259887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,3,0.012603199481964112
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,7,0.012342400103807449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,7,0.012667199969291687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,15,0.011804799735546111
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,15,0.012656000256538392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,31,0.01210559979081154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,31,0.012342400103807449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,63,0.011902400106191636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,63,0.01342879980802536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,127,0.012009599804878235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,127,0.012649600207805634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,255,0.01443839967250824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,255,0.013507199287414551
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,511,0.01769919991493225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,511,0.018193599581718446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,1023,0.01701280027627945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,1023,0.017790399491786957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,float16,2047,0.017543999850749968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,1,128,1,float16,fp8,2047,0.019592000544071196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,1,0.011830399930477142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,1,0.01268640011548996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,3,0.01178399994969368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,3,0.01260959953069687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,7,0.012481600046157837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,7,0.012670400738716125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,15,0.011966399848461151
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,15,0.012518399953842163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,31,0.011819200217723846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,31,0.012617599964141846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,63,0.012195199728012085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,63,0.012700800597667695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,127,0.011851199716329575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,127,0.012598399817943574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,255,0.01383039951324463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,255,0.015193599462509155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,511,0.016620799899101257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,511,0.018068799376487733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,1023,0.01661919951438904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,1023,0.017478400468826295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,float16,2047,0.019180800020694732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,2,128,1,float16,fp8,2047,0.018036800622940063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,1,0.011854399740695954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,1,0.012604799866676331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,3,0.011700800061225891
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,3,0.013305599987506866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,7,0.012001600116491318
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,15,0.011879999935626984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,7,0.012652799487113953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,15,0.012505599856376648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,31,0.012545600533485413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,31,0.012880000472068786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,63,0.011908800154924393
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,63,0.01268479973077774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,127,0.011990399658679962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,127,0.01334560066461563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,255,0.013841600716114044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,255,0.014590400457382201
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,511,0.01663520038127899
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,511,0.018489600718021394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,1023,0.017558400332927705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,1023,0.017492799460887908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,float16,2047,0.020467199385166168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,1,0.01234079971909523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,4,128,1,float16,fp8,2047,0.02107200026512146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,1,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,3,0.012080000340938568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,3,0.012619200348854064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,7,0.01189119964838028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,7,0.012936000525951386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,15,0.01183359995484352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,15,0.012727999687194824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,31,0.012041600048542022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,63,0.01188800036907196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,63,0.0131632000207901
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,31,0.01276959925889969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,127,0.012076800316572189
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,255,0.013831999897956849
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,127,0.012905600666999816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,255,0.014707200229167938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,511,0.017211200296878816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,511,0.018406400084495546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,1023,0.019998399913311003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,1023,0.02067520022392273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,float16,2047,0.037520000338554384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,96,8,128,1,float16,fp8,2047,0.027369600534439088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,1,0.06401280164718628
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,1,0.06627839803695679
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,3,0.06329119801521302
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,7,0.06352000236511231
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,3,0.0656544029712677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,7,0.0661952018737793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,15,0.06448959708213806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,15,0.06691200137138367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,31,0.06526880264282227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,31,0.06753919720649719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,fp8,63,0.07070879936218262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,1,128,1,float16,float16,63,0.06809920072555542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,1,0.09054080247879029
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,3,0.09006239771842957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,1,0.10026880502700805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,3,0.10058399438858032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,7,0.09002879858016968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,15,0.09088640213012696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,7,0.10111199617385865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,15,0.10122560262680054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,31,0.09307360053062438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,31,0.10246560573577881
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,float16,63,0.09549599885940552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,2,128,1,float16,fp8,63,0.10578080415725707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,1,0.14638880491256714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,3,0.1461135983467102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,1,0.17100800275802613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,3,0.1705888032913208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,7,0.1476367950439453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,15,0.14677439928054808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,7,0.1707535982131958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,15,0.18844799995422362
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,31,0.14906400442123413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,31,0.17385599613189698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,float16,63,0.15204319953918458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,4,128,1,float16,fp8,63,0.17625919580459595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,1,0.26388161182403563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,1,0.3458064079284668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,3,0.2642767906188965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,7,0.2630352020263672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,3,0.31031999588012693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,7,0.3125616073608398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,15,0.31374239921569824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,31,0.26776158809661865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,15,0.3160048007965088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,31,0.3130111932754517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,float16,63,0.2687839984893799
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,1,0.11070719957351685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,1,0.1209488034248352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,3,0.11067839860916137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,7,0.11180319786071777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,3,0.12144160270690918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,96,8,128,1,float16,fp8,63,0.40603199005126955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,7,0.12010079622268677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,15,0.11242879629135132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,15,0.12169439792633056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,1,128,1,float16,float16,31,0.1135632038116455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,1,128,1,float16,fp8,31,0.12260799407958985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,1,0.18527040481567383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,3,0.16302560567855834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,1,0.20338881015777588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,3,0.18660000562667847
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,7,0.1629439949989319
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,7,0.18751360177993776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,15,0.16425600051879882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,15,0.18711839914321898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,2,128,1,float16,float16,31,0.20509920120239258
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,2,128,1,float16,fp8,31,0.18948960304260254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,1,0.27409920692443845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,1,0.32609920501708983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,3,0.2734544038772583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,3,0.3229151964187622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,15,0.27476000785827637
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,7,0.34993278980255127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,7,0.32501280307769775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,4,128,1,float16,float16,31,0.2783040046691895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,15,0.32767839431762696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,4,128,1,float16,fp8,31,0.32741119861602785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,1,0.5044655799865723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,3,0.5067327976226806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,7,0.5044000148773193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,3,0.6047791957855224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,1,0.7968592166900634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,7,0.6077055931091309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,15,0.505401611328125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,1,0.012137600034475327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,1,0.01589120030403137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,3,0.012171199917793274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,3,0.012289600074291229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,7,0.01196800023317337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,7,0.015548799932003022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,15,0.012033600360155106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,15,0.012212800234556198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,31,0.012057600170373916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,15,0.600816011428833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,31,0.015516799688339234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,8,128,1,float16,float16,31,0.5092192173004151
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,63,0.012011200189590454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,63,0.012249600142240524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,127,0.01199520006775856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,255,0.016816000640392303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,96,8,128,1,float16,fp8,31,0.6055103778839112
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,127,0.012673600018024445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,511,0.016652800142765045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,511,0.017716799676418305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,1023,0.017580799758434296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,255,0.014483200013637542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,1023,0.021379199624061585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,1,0.01191679984331131
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,float16,2047,0.019681599736213685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,1,0.012240000069141388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,1,128,1,float16,fp8,2047,0.01919520050287247
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,3,0.011985599994659424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,3,0.014790399372577668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,7,0.011873599886894227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,7,0.012697599828243256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,15,0.011612799763679505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,15,0.012744000554084778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,31,0.012017600238323212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,31,0.014819200336933135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,63,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,63,0.012068799883127212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,127,0.011816000193357467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,127,0.012856000661849975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,255,0.01387840062379837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,255,0.01736319959163666
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,511,0.016697600483894348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,511,0.017824000120162962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,1023,0.018382400274276733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,1,0.011847999691963196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,1,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,float16,2047,0.021104000508785248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,1023,0.0177824005484581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,2,128,1,float16,fp8,2047,0.0234592005610466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,3,0.012044800072908401
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,3,0.012582400441169738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,7,0.011761599779129028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,7,0.012671999633312225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,31,0.011796800047159195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,15,0.012057600170373916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,31,0.01250240057706833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,63,0.011622399836778641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,15,0.013782399892807006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,63,0.012886400520801543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,127,0.011841599643230439
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,255,0.014379200339317322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,255,0.013633599877357483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,511,0.018276800215244294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,127,0.01286720037460327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,511,0.018296000361442567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,1023,0.020022399723529816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,1023,0.02040960043668747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,float16,2047,0.03828159868717194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,1,0.012054400146007537
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,4,128,1,float16,fp8,2047,0.029860800504684447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,1,0.012824000418186187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,3,0.011767999827861786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,3,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,7,0.011774399876594543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,15,0.01215519979596138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,7,0.012912000715732574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,31,0.011695999652147293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,31,0.012462399899959564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,15,0.013312000036239623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,63,0.011806400120258331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,63,0.01284639984369278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,127,0.012216000258922577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,127,0.012598399817943574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,255,0.013676799833774567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,255,0.015299199521541596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,511,0.01796800047159195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,511,0.018380799889564516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,1023,0.03608959913253784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,1023,0.025814399123191833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,1,0.012870399653911591
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,float16,2047,0.057329601049423216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,1,0.013916799426078796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,96,8,128,1,float16,fp8,2047,0.04244160056114197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,3,0.013652800023555756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,7,0.01284320056438446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,15,0.013156799972057343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,3,0.013145600259304047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,7,0.013638399541378021
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,15,0.014364799857139588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,31,0.013699199259281158
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,31,0.013099199533462525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,63,0.013262400031089782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,63,0.013689599931240082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,127,0.013513599336147309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,127,0.01441120058298111
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,255,0.01526080071926117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,255,0.015475200116634369
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,511,0.017763200402259826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,511,0.01945919990539551
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,1023,0.023739199340343475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,1023,0.02330559939146042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,2047,0.030031999945640563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,2047,0.04195840060710907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,1,0.013307200372219085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,fp8,4095,0.04392319917678833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,1,128,1,float16,float16,4095,0.0632431983947754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,1,0.013823999464511872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,3,0.013212800025939941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,3,0.013644799590110779
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,7,0.013385599851608277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,7,0.013991999626159667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,31,0.012972800433635712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,15,0.01316480040550232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,15,0.014425599575042724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,31,0.014028799533843995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,63,0.012980799376964568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,63,0.014374400675296783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,127,0.013398399949073792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,127,0.013784000277519226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,255,0.015668800473213194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,255,0.015614399313926696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,511,0.018648000061511995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,511,0.019963200390338897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,1023,0.03844960033893585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,1023,0.027428799867630006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,2047,0.04143039882183075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,2047,0.06010079979896545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,1,0.016967999935150146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,fp8,4095,0.06796159744262695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,1,0.018067200481891633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,3,0.016924799978733064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,3,0.019411200284957887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,2,128,1,float16,float16,4095,0.10297919511795044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,7,0.018131199479103088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,7,0.017371200025081635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,15,0.01677599996328354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,15,0.01805119961500168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,31,0.018004800379276275
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,63,0.01674399971961975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,31,0.01828159987926483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,127,0.016899199783802034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,63,0.018649600446224213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,127,0.018028800189495087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,255,0.022196799516677856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,255,0.02189760059118271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,511,0.03888320028781891
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,511,0.030747199058532716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,1023,0.06290720105171203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,1023,0.0467056006193161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,2047,0.06908479928970337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,2047,0.10576319694519043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,1,0.024135999381542206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,1,0.027238398790359497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,float16,4095,0.18816640377044677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,3,0.02386240065097809
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,4,128,1,float16,fp8,4095,0.11962239742279053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,3,0.029528000950813295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,7,0.02407840043306351
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,7,0.027321600914001466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,15,0.026657599210739135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,15,0.02447360008955002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,63,0.024377599358558655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,31,0.02969599962234497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,31,0.024116800725460054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,63,0.027403199672698976
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,127,0.025811201333999632
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,127,0.027433601021766663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,255,0.0440064013004303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,255,0.0394320011138916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,511,0.06369919776916504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,511,0.05297600030899048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,1023,0.10771360397338867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,1023,0.0782800018787384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,1,0.011296000331640244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,2047,0.13375840187072754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,1,0.012219200283288956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,3,0.011323200166225433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,2047,0.19061119556427003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,7,0.011448000371456147
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,3,0.012992000579833985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,7,0.012174399942159653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,fp8,4095,0.2272752046585083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,15,0.01149279996752739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,31,0.011416000127792359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,15,0.012299200147390365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,31,0.013020800054073333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,64,8,128,1,float16,float16,4095,0.3562175989151001
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,63,0.011385600268840789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,127,0.011526399850845337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,255,0.013910399377346038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,127,0.012188799679279327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,63,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,511,0.015958400070667268
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,511,0.01804639995098114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,1023,0.016883200407028197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,255,0.014124800264835358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,1023,0.017067199945449828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,2047,0.016131199896335602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,2047,0.0173007994890213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,float16,4095,0.01684480011463165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,1,128,1,float16,fp8,4095,0.018739199638366698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,1,0.011559999734163284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,1,0.01207519993185997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,3,0.01141439974308014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,3,0.01231200024485588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,7,0.011267200112342834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,7,0.01297920048236847
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,15,0.011596799641847611
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,31,0.011283200234174728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,15,0.012091200053691863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,63,0.011559999734163284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,31,0.01239359974861145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,63,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,127,0.012174399942159653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,127,0.012068799883127212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,255,0.012913599610328674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,255,0.014302399754524232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,511,0.016232000291347505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,511,0.019198399782180787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,1023,0.016545599699020384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,1023,0.015518400073051452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,2047,0.01613599956035614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,float16,4095,0.017657600343227386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,2047,0.0170864000916481
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,1,0.011401599645614624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,2,128,1,float16,fp8,4095,0.0173552006483078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,1,0.012256000190973282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,3,0.012039999663829803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,7,0.011324799805879592
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,3,0.012363199889659882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,15,0.011526399850845337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,7,0.011947199702262878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,15,0.012782399356365205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,31,0.011435200273990632
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,63,0.011385600268840789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,31,0.012337599694728852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,127,0.011774399876594543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,63,0.012270399928092956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,127,0.012083200365304947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,255,0.012916800379753113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,255,0.014310400187969207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,511,0.017867200076580048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,511,0.016283200681209566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,1023,0.015889599919319153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,1023,0.016654400527477263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,2047,0.01608159989118576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,float16,4095,0.01732800006866455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,2047,0.016982400417327882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,1,0.011398400366306304
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,4,128,1,float16,fp8,4095,0.01788160055875778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,1,0.012225600332021714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,3,0.011598400026559829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,7,0.011423999816179276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,3,0.012204799801111221
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,7,0.012647999823093415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,15,0.011588799953460693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,15,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,31,0.011468800157308579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,31,0.01218079999089241
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,63,0.011671999841928482
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,127,0.0117807999253273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,63,0.012331199645996094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,127,0.012139199674129486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,255,0.013022400438785553
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,255,0.014248000085353851
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,511,0.01790879964828491
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,511,0.016344000399112702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,1023,0.015675200521945952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,1023,0.016843199729919434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,2047,0.016463999450206757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,4095,0.017752000689506532
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,float16,4095,0.018323199450969697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,64,8,128,1,float16,fp8,2047,0.017211200296878816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,1,0.011529599875211715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,1,0.012272000312805176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,3,0.011548800021409988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,7,0.011436799913644791
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,3,0.012772800028324127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,15,0.011460799723863602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,7,0.012415999919176102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,15,0.012337599694728852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,31,0.01178399994969368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,31,0.012520000338554382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,63,0.011627200245857238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,63,0.01234079971909523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,127,0.011407999694347382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,127,0.012449599802494049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,255,0.01348160058259964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,255,0.01427839994430542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,511,0.016176000237464905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,511,0.017936000227928163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,1023,0.016019199788570405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,1023,0.017416000366210938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,2047,0.01656160056591034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,2047,0.0176144003868103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,1,0.011662399768829346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,float16,4095,0.016998399794101716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,1,128,1,float16,fp8,4095,0.018057599663734436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,1,0.012355200201272964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,3,0.011559999734163284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,3,0.012190400063991547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,7,0.011566399782896041
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,7,0.012324800342321396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,15,0.011460799723863602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,15,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,31,0.011358399689197541
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,31,0.012363199889659882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,63,0.01162400022149086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,63,0.012275200337171555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,127,0.011508800089359283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,127,0.012108799815177918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,255,0.013273599743843078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,255,0.014094400405883788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,511,0.016051200032234193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,511,0.018086400628089905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,1023,0.01685280054807663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,1023,0.016044799983501435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,2047,0.016180799901485445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,2047,0.017265599966049195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,float16,4095,0.01687999963760376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,2,128,1,float16,fp8,4095,0.017875200510025023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,1,0.011457599699497223
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,1,0.012200000137090683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,3,0.011443199962377549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,3,0.012449599802494049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,7,0.011367999762296677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,7,0.012167999893426895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,15,0.011419200152158738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,15,0.012383999675512314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,31,0.011475200206041336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,31,0.012230399996042252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,63,0.011430399864912033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,63,0.012363199889659882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,127,0.011659199744462967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,127,0.012166400253772736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,255,0.013198399543762207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,255,0.014292800426483154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,511,0.017774400115013123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,511,0.016307200491428375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,1023,0.01589120030403137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,1023,0.01713919937610626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,2047,0.01650400012731552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,2047,0.017308799922466277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,float16,4095,0.018272000551223754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,4,128,1,float16,fp8,4095,0.018016000092029572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,1,0.011476799845695496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,1,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,3,0.0117807999253273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,7,0.011500799655914306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,3,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,7,0.012302400171756744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,15,0.011795199662446975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,15,0.012251199781894683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,31,0.011675199866294861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,31,0.012363199889659882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,63,0.011806400120258331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,63,0.012111999839544297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,127,0.011795199662446975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,127,0.012430399656295776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,255,0.013396799564361572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,255,0.0143327996134758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,511,0.016148799657821657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,1023,0.015892800688743592
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,511,0.018192000687122345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,1023,0.01717280000448227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,2047,0.017399999499320983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,2047,0.01777759939432144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,float16,4095,0.02147520035505295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,64,8,128,1,float16,fp8,4095,0.021297599375247955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,1,0.012574400007724761
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,1,0.013252800703048706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,3,0.01250240057706833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,3,0.013363200426101684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,7,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,7,0.01348000019788742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,15,0.012593600153923034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,15,0.013238400220870972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,31,0.013315199315547943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,31,0.012535999715328216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,63,0.012649600207805634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,63,0.013489599525928497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,127,0.01276639997959137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,255,0.014499199390411378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,127,0.013318400084972381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,255,0.015275199711322785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,511,0.0175135999917984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,511,0.018827199935913086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,1023,0.019548800587654114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,1023,0.018956799805164338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,2047,0.023443199694156647
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,2047,0.022574399411678315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,1,0.012667199969291687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,float16,4095,0.040759998559951785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,1,0.013148799538612366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,1,128,1,float16,fp8,4095,0.030019199848175047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,3,0.012587200105190276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,7,0.012600000202655792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,3,0.013350400328636169
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,7,0.0133200004696846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,15,0.012590399384498597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,31,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,31,0.013422399759292603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,63,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,15,0.013388800621032714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,63,0.013358399271965027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,127,0.012763200700283051
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,127,0.013497599959373474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,255,0.014552000164985656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,255,0.015244799852371215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,511,0.01759839951992035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,511,0.01886879950761795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,1023,0.021955199539661407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,1023,0.021908800303936004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,2047,0.02877599895000458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,2047,0.0393312007188797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,float16,4095,0.06066719889640808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,1,0.012646399438381195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,1,0.013406400382518769
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,3,0.012673600018024445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,7,0.012539200484752655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,3,0.013420799374580383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,2,128,1,float16,fp8,4095,0.04327360093593598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,7,0.013327999413013459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,15,0.012753599882125854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,15,0.013371199369430542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,31,0.012726399302482604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,31,0.013417600095272065
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,63,0.012611199915409089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,63,0.01356479972600937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,127,0.012839999794960023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,127,0.013483199477195739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,255,0.014670400321483612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,255,0.015345600247383118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,511,0.01773280054330826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,511,0.019064000248908995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,1023,0.03707520067691803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,1023,0.026939201354980468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,2047,0.058462399244308474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,2047,0.041140800714492796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,1,0.016318400204181672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,float16,4095,0.10121599435806275
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,1,0.017723199725151063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,4,128,1,float16,fp8,4095,0.06743040084838867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,3,0.01652960032224655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,3,0.017800000309944154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,7,0.016411200165748596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,7,0.017851200699806214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,15,0.01650400012731552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,15,0.01780800074338913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,31,0.016616000235080718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,31,0.017977599799633027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,63,0.016484799981117248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,63,0.017947199940681457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,127,0.017862400412559508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,127,0.01664479970932007
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,255,0.02080480009317398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,255,0.021793599426746368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,511,0.029766398668289184
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,511,0.0369376003742218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,1023,0.060177600383758544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,1023,0.04626879990100861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,2047,0.1028656005859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,1,0.011649599671363831
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,2047,0.06977760195732116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,3,0.011603199690580369
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,1,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,fp8,4095,0.12072639465332032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,64,8,128,1,float16,float16,4095,0.1856160044670105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,3,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,7,0.011659199744462967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,7,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,31,0.011660800129175187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,15,0.011683200299739838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,31,0.012572799623012543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,63,0.01162559986114502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,15,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,63,0.012387199699878693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,127,0.01170239970088005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,127,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,255,0.013191999495029449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,255,0.014412799477577209
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,511,0.01621599942445755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,511,0.018113599717617036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,1023,0.016579200327396394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,1023,0.017390400171279907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,2047,0.016606399416923524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,2047,0.017692799866199493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,float16,4095,0.017615999281406402
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,1,0.011737599968910217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,1,128,1,float16,fp8,4095,0.018438400328159334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,1,0.01234399974346161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,3,0.011664000153541566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,3,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,7,0.011521600186824799
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,7,0.012385600060224534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,15,0.011726400256156922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,15,0.01308799982070923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,31,0.01175680011510849
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,31,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,63,0.011587200313806533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,127,0.011683200299739838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,63,0.012332800030708312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,127,0.013131199777126313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,255,0.01329600065946579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,255,0.014393599331378936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,511,0.016288000345230102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,511,0.017936000227928163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,1023,0.01616320013999939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,1023,0.018348799645900728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,2047,0.016763199865818024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,2047,0.017419199645519256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,float16,4095,0.018545599281787874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,1,0.011788800358772278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,2,128,1,float16,fp8,4095,0.018116800487041472
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,1,0.013433599472045898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,3,0.01180799975991249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,3,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,7,0.011716800183057785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,7,0.012414400279521943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,15,0.012665599584579468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,15,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,31,0.011529599875211715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,31,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,63,0.011644800007343293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,63,0.013273599743843078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,127,0.0117296002805233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,127,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,255,0.01348160058259964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,255,0.014454400539398194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,511,0.017076799273490907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,511,0.018251200020313264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,1023,0.01624480038881302
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,1023,0.017286400496959686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,2047,0.017955200374126436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,2047,0.018310399353504182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,1,0.011985599994659424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,float16,4095,0.02154400050640106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,4,128,1,float16,fp8,4095,0.021465599536895752
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,1,0.012598399817943574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,3,0.012223999947309494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,3,0.012863999605178833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,7,0.01266079992055893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,7,0.011840000003576278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,15,0.013091200590133667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,31,0.01196800023317337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,15,0.011806400120258331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,31,0.012656000256538392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,63,0.012012799829244613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,63,0.013147200644016265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,127,0.012646399438381195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,127,0.012060800194740295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,255,0.013966399431228637
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,255,0.01478240042924881
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,511,0.01698880046606064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,1023,0.017769600450992584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,511,0.018161599338054658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,1023,0.01727679967880249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,2047,0.020160000026226043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,2047,0.021718400716781616
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,1,0.014670400321483612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,fp8,4095,0.02773120105266571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,64,8,128,1,float16,float16,4095,0.03733119964599609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,1,0.015384000539779664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,3,0.015000000596046448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,3,0.01583999991416931
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,7,0.014662399888038635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,15,0.014535999298095703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,7,0.015647999942302704
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,15,0.01592639982700348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,31,0.0147599995136261
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,31,0.01544319987297058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,63,0.01470080018043518
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,63,0.016065600514411926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,127,0.015337599813938141
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,127,0.01603199988603592
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,255,0.016705599427223206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,255,0.0174687996506691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,511,0.0213919997215271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,511,0.021408000588417055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,1023,0.04087199866771698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,1023,0.030287998914718627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,1,0.018771199882030486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,float16,2047,0.0618511974811554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,1,0.02008959949016571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,1,128,1,float16,fp8,2047,0.043977600336074826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,3,0.01866399943828583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,7,0.018713599443435668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,3,0.020172800123691558
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,15,0.01860480010509491
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,7,0.019868800044059755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,31,0.01866399943828583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,31,0.020427200198173522
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,15,0.01971520036458969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,63,0.018675200641155243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,63,0.020207999646663664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,127,0.020108799636363982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,127,0.018475200235843658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,255,0.024128000438213348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,511,0.04052320122718811
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,255,0.02364159971475601
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,511,0.03252480030059814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,1023,0.04877760112285614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,1023,0.06555520296096802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,1,0.025923201441764833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,1,0.029108801484107973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,fp8,2047,0.07245759963989258
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,2,128,1,float16,float16,2047,0.10781439542770385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,3,0.02600319981575012
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,7,0.02600800096988678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,3,0.029179200530052185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,7,0.028947201371192933
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,15,0.02608479857444763
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,15,0.028863999247550964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,31,0.025812798738479616
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,63,0.02627359926700592
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,31,0.02927039861679077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,63,0.029068800806999206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,127,0.02805280089378357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,127,0.028960001468658448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,255,0.04542239904403687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,255,0.03794719874858856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,511,0.0670687973499298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,511,0.05467360019683838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,1023,0.11232479810714721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,1,0.04039359986782074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,1023,0.08045920133590698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,fp8,2047,0.1288831949234009
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,3,0.04043200016021729
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,1,0.046249601244926455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,4,128,1,float16,float16,2047,0.19427679777145385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,3,0.046500799059867856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,7,0.04650239944458008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,7,0.040468800067901614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,15,0.04020799994468689
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,31,0.04051199853420258
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,15,0.04667679965496063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,31,0.04660640060901642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,63,0.042124798893928526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,63,0.04651040136814118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,127,0.04830879867076874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,127,0.04883359968662262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,255,0.07532960176467896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,255,0.06600480079650879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,511,0.11537120342254639
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,511,0.09478240013122559
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,1023,0.14605920314788817
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,1023,0.19905760288238525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,1,0.021451200544834136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,1,0.022448000311851502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,7,0.021430400013923646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,3,0.02086080014705658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,fp8,2047,0.23930718898773193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,7,0.022409600019454957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,3,0.02264160066843033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,64,8,128,1,float16,float16,2047,0.3675839900970459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,15,0.02131839990615845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,15,0.0226160004734993
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,31,0.022303999960422517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,31,0.02091359943151474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,63,0.020921599864959717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,127,0.022094400227069856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,127,0.022763200104236603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,255,0.02656640112400055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,255,0.0305759996175766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,63,0.02295680046081543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,float16,511,0.045491200685501096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,1,0.02828960120677948
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,1,0.031174400448799135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,3,0.02799679934978485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,3,0.03136959969997406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,1,128,1,float16,fp8,511,0.036457601189613345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,7,0.02810879945755005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,7,0.03131040036678314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,15,0.028198400139808656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,15,0.031142398715019226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,31,0.028225600719451904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,31,0.031937599182128906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,63,0.02823840081691742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,127,0.0339792013168335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,63,0.03133760094642639
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,127,0.03115679919719696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,255,0.04813440144062042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,255,0.0411871999502182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,fp8,511,0.056775999069213864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,1,0.04307200014591217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,2,128,1,float16,float16,511,0.07114239931106567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,1,0.04874880015850067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,3,0.04294559955596924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,7,0.042535999417304994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,7,0.049051201343536376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,3,0.04937120079994202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,15,0.042788800597190854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,15,0.04877119958400726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,31,0.042715200781822206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,31,0.04904800057411194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,63,0.047417598962783816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,127,0.053281599283218385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,63,0.04941760003566742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,255,0.0762607991695404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,127,0.0525551974773407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,255,0.06893119812011719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,float16,511,0.11902079582214356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,1,0.07110559940338135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,1,0.08366720080375671
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,3,0.07074080109596252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,4,128,1,float16,fp8,511,0.09779679775238037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,3,0.08378720283508301
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,7,0.08421440124511718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,7,0.07131839990615844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,15,0.0711359977722168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,15,0.0836143970489502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,31,0.07097920179367065
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,31,0.0841152012348175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,63,0.0783728003501892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,127,0.08869919776916504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,63,0.08847360014915466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,127,0.09701279997825622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,1,0.03441919982433319
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,255,0.11969759464263915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,255,0.1314128041267395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,1,0.03766399919986725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,3,0.03443360030651092
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,3,0.037363201379776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,fp8,511,0.17974239587783813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,7,0.034308800101280214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,64,8,128,1,float16,float16,511,0.2160720109939575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,7,0.03751200139522552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,15,0.03457759916782379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,15,0.037492799758911136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,31,0.03471199870109558
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,31,0.03729760050773621
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,127,0.043028798699378965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,63,0.03494719862937927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,63,0.03746879994869232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,127,0.03758719861507416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,float16,255,0.0557807981967926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,1,128,1,float16,fp8,255,0.04795680046081543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,1,0.048871999979019164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,1,0.054287999868392944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,3,0.04832479953765869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,3,0.05418720245361328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,7,0.04891999959945679
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,7,0.05435360074043274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,15,0.054448002576828004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,15,0.048875200748443606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,31,0.04876640141010284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,31,0.05421919822692871
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,63,0.05498239994049072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,63,0.05548160076141358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,127,0.06157919764518738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,127,0.06067039966583252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,float16,255,0.08266879916191101
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,1,0.07724320292472839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,2,128,1,float16,fp8,255,0.07339839935302735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,1,0.08897439837455749
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,3,0.07731999754905701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,3,0.08966559767723084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,7,0.07630079984664917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,7,0.08894720077514648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,15,0.07719359993934631
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,15,0.08913919925689698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,31,0.08082559704780579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,31,0.08994560241699219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,63,0.08488479852676392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,63,0.09431679844856262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,127,0.088155198097229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,127,0.09921919703483581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,float16,255,0.1299504041671753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,4,128,1,float16,fp8,255,0.12535840272903442
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,1,0.1300320029258728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,1,0.15599520206451417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,3,0.12961440086364745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,3,0.15785759687423706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,7,0.13213759660720825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,7,0.15706559419631957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,15,0.13092479705810547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,31,0.14124959707260132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,15,0.1562543988227844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,31,0.16289440393447877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,63,0.14449280500411987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,63,0.16750880479812622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,1,0.01178240031003952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,127,0.15331519842147828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,127,0.17944960594177245
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,1,0.012452799826860428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,3,0.011710400134325028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,float16,255,0.22866559028625488
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,3,0.012585599720478059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,7,0.012595200538635254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,15,0.011769600212574005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,64,8,128,1,float16,fp8,255,0.22503199577331542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,7,0.011687999963760376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,31,0.011873599886894227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,31,0.012510399520397186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,63,0.011814399808645248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,15,0.012606400251388549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,127,0.011772800236940384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,127,0.012511999905109405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,63,0.012328000366687774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,255,0.013422399759292603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,255,0.014502400159835815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,511,0.01652960032224655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,511,0.017929600179195405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,1023,0.01650879979133606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,1023,0.017377600073814392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,2047,0.01796640008687973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,2047,0.017081600427627564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,float16,4095,0.018775999546051025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,1,0.011847999691963196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,1,128,1,float16,fp8,4095,0.01857440024614334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,1,0.01239520013332367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,3,0.011910399794578553
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,3,0.012715199589729309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,7,0.012008000165224075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,7,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,15,0.011812800168991089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,15,0.012611199915409089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,31,0.01194240003824234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,31,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,63,0.011953599750995636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,63,0.012615999579429627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,127,0.012006399780511856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,127,0.012705600261688233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,255,0.01358560025691986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,255,0.014547200500965118
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,511,0.016396799683570863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,511,0.01818079948425293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,1023,0.016579200327396394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,1023,0.017308799922466277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,2047,0.017800000309944154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,2047,0.01815200001001358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,1,0.01204800009727478
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,fp8,4095,0.02182080000638962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,1,0.012785600125789642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,2,128,1,float16,float16,4095,0.022433599829673766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,3,0.012031999975442886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,3,0.012779200077056884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,7,0.012054400146007537
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,15,0.012751999497413635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,31,0.012009599804878235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,7,0.013017599284648896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,15,0.012067200243473053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,31,0.012771199643611907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,63,0.012011200189590454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,63,0.012703999876976013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,127,0.012388800084590913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,127,0.012856000661849975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,255,0.013793599605560303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,511,0.016808000206947327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,255,0.014908799529075622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,511,0.01823039948940277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,1023,0.01826400011777878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,1023,0.01751679927110672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,2047,0.020499199628829956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,float16,4095,0.038206401467323306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,1,0.012166400253772736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,2047,0.02119999974966049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,4,128,1,float16,fp8,4095,0.027001601457595826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,1,0.012619200348854064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,3,0.011992000043392181
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,3,0.012664000689983367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,15,0.011931200325489045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,7,0.012145599722862244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,15,0.012995199859142303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,7,0.012753599882125854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,31,0.012185599654912949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,31,0.012836800515651703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,63,0.012052799761295318
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,63,0.012884800136089326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,127,0.012268800288438797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,127,0.012942400574684144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,255,0.014060799777507783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,255,0.014791999757289887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,511,0.01690720021724701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,511,0.01833759993314743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,1023,0.019675199687480927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,1023,0.021096000075340272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,2047,0.02691200077533722
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,2047,0.037108799815177916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,float16,4095,0.058361601829528806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,1,0.05733280181884766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,64,8,128,1,float16,fp8,4095,0.04135519862174988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,1,0.06179839968681335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,3,0.05635039806365967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,7,0.057004797458648684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,3,0.06175199747085571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,7,0.06165599822998047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,15,0.05811039805412292
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,15,0.062193602323532104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,31,0.06285600066184997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,31,0.06030880212783814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,63,0.0645359992980957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,63,0.06611520051956177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,1,0.084060800075531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,fp8,127,0.06904640197753906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,1,128,1,float16,float16,127,0.07024639844894409
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,3,0.08399199843406677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,1,0.09512479901313782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,3,0.09527040123939515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,7,0.08503519892692565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,7,0.09540640115737915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,15,0.09597280025482177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,15,0.08700799942016602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,31,0.08912799954414367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,31,0.09821919798851013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,63,0.09303200244903564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,63,0.10236480236053466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,float16,127,0.10084960460662842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,2,128,1,float16,fp8,127,0.1060528039932251
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,1,0.14091999530792237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,3,0.1411504030227661
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,1,0.1624127984046936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,3,0.1626479983329773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,7,0.14277119636535646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,7,0.16307519674301146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,15,0.14426399469375611
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,15,0.16549439430236818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,31,0.14637759923934937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,31,0.17074559926986693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,63,0.14918559789657593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,63,0.1727839946746826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,float16,127,0.16090240478515624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,4,128,1,float16,fp8,127,0.1784832000732422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,1,0.2576911926269531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,1,0.292579197883606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,3,0.2573535919189453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,7,0.2608448028564453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,3,0.35243520736694334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,7,0.29857120513916013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,15,0.2608720064163208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,31,0.26246559619903564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,15,0.30720479488372804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,1,0.10574400424957275
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,63,0.3167824029922485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,31,0.3153264045715332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,63,0.31635360717773436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,float16,127,0.2879040002822876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,1,0.11599839925765991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,64,8,128,1,float16,fp8,127,0.3449359893798828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,3,0.12761600017547609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,3,0.11663360595703125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,7,0.10509120225906372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,7,0.11524159908294677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,15,0.10651359558105469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,15,0.11579359769821167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,31,0.12885440587997438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,31,0.11781280040740967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,float16,63,0.11143840551376342
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,1,128,1,float16,fp8,63,0.11913759708404541
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,1,0.15877120494842528
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,1,0.18105119466781616
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,3,0.19640159606933594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,7,0.15816800594329833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,3,0.18313759565353394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,7,0.18248000144958496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,15,0.1589344024658203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,15,0.18208800554275512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,31,0.1975823998451233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,31,0.1848415970802307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,fp8,63,0.18944799900054932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,2,128,1,float16,float16,63,0.1666751980781555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,1,0.26881120204925535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,1,0.32052481174468994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,3,0.2682543992996216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,7,0.2690288066864014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,15,0.2705696105957031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,3,0.40013761520385743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,7,0.3224495887756348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,15,0.3215199947357178
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,31,0.27344479560852053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,31,0.32266080379486084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,float16,63,0.27775039672851565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,4,128,1,float16,fp8,63,0.3285808086395264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,3,0.4968128204345703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,1,0.6187024116516113
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,1,0.6052671909332276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,3,0.6029151916503906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,7,0.4964752197265625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,7,0.6021312236785888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,15,0.49945759773254395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,1,0.012049599736928939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,1,0.01279039978981018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,3,0.01207519993185997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,3,0.012545600533485413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,7,0.011963199824094772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,7,0.012531200051307678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,15,0.011974400281906128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,15,0.6002816200256348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,31,0.6188464164733887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,31,0.5980463981628418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,float16,63,0.5086063861846923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,15,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,31,0.01199679970741272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,31,0.012483199685811996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,64,8,128,1,float16,fp8,63,0.605017614364624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,63,0.014545600116252898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,127,0.01279360055923462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,255,0.013348799943923951
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,63,0.012267199903726577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,255,0.014375999569892883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,127,0.01210239976644516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,511,0.02234400063753128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,511,0.01653759926557541
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,1023,0.016820800304412842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,2047,0.018246400356292724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,2047,0.018593600392341612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,1,0.01213119998574257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,1,0.012547199428081513
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,float16,4095,0.02245440036058426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,4095,0.026547199487686156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,3,0.011750400066375732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,7,0.01204639971256256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,3,0.012375999987125397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,7,0.012649600207805634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,1,128,1,float16,fp8,1023,0.017715199291706084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,15,0.015438400208950043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,15,0.011868800222873687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,31,0.011695999652147293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,31,0.012403199821710587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,63,0.01191679984331131
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,63,0.012972800433635712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,127,0.01467519998550415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,127,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,255,0.013902400434017182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,255,0.01456640064716339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,511,0.016808000206947327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,511,0.01838880032300949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,1023,0.02166240066289902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,1023,0.017267200350761413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,2047,0.020496000349521638
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,2047,0.020720000565052032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,float16,4095,0.03836480081081391
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,1,0.014655999839305878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,2,128,1,float16,fp8,4095,0.028428798913955687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,1,0.012788799405097962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,3,0.01199520006775856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,3,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,7,0.012132800370454788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,7,0.014164799451828003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,15,0.01279039978981018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,15,0.01226240023970604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,31,0.01196800023317337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,31,0.01266079992055893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,63,0.01207199990749359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,63,0.014155200123786927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,127,0.012111999839544297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,255,0.013928000628948212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,127,0.012964800000190735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,255,0.014776000380516052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,511,0.016835199296474458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,511,0.020235200226306916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,1023,0.019952000677585603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,1023,0.020828799903392793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,2047,0.027404800057411194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,2047,0.03811199963092804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,float16,4095,0.05967040061950683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,1,0.011964800208806992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,1,0.01289760023355484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,4,128,1,float16,fp8,4095,0.04360480010509491
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,3,0.01204639971256256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,15,0.012425599992275238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,7,0.012830400466918945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,7,0.012064000219106674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,3,0.013124799728393555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,15,0.012910400331020356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,31,0.012729600071907043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,31,0.012865599989891053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,63,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,63,0.012768000364303589
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,127,0.01231039986014366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,127,0.013009600341320038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,255,0.014001600444316864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,255,0.014932799339294433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,511,0.01802240014076233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,511,0.01887679994106293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,1023,0.026276800036430358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,1023,0.036164799332618715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,2047,0.04050720036029816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,1,0.013214400410652161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,2047,0.057473599910736084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,1,0.01374559998512268
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,float16,4095,0.10025919675827026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,3,0.012860800325870513
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,64,8,128,1,float16,fp8,4095,0.06718559861183167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,7,0.014071999490261078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,3,0.013524800539016724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,7,0.01316000074148178
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,15,0.012886400520801543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,31,0.012836800515651703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,63,0.01321759968996048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,15,0.013683199882507324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,63,0.01372320055961609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,31,0.014179199934005737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,127,0.013470399379730224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,127,0.013761599361896516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,255,0.014793600142002105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,255,0.015886400640010834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,511,0.019196799397468566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,511,0.017617599666118623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,2047,0.040031999349594116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,1023,0.022273600101470947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,1023,0.021481600403785706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,2047,0.028702399134635924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,4095,0.06092960238456726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,4095,0.042392000555992126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,float16,8191,0.10435999631881714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,1,0.01326719969511032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,1,0.013689599931240082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,3,0.013179199397563934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,1,128,1,float16,fp8,8191,0.06866400241851807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,7,0.012899200618267059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,3,0.014052799344062806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,15,0.01327199935913086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,7,0.013665600121021271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,15,0.013798399269580841
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,31,0.01334560066461563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,31,0.014323200285434722
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,63,0.01327040046453476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,63,0.013963200151920319
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,127,0.013129599392414093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,127,0.013702400028705597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,255,0.015254400670528412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,511,0.018751999735832213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,255,0.016246399283409117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,511,0.019566400349140166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,1023,0.03719359934329987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,1023,0.02715519964694977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,2047,0.059166401624679565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,2047,0.04161440134048462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,4095,0.10242240428924561
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,1,0.016935999691486358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,4095,0.06628000140190124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,1,0.018452799320220946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,fp8,8191,0.11538560390472412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,3,0.018508799374103546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,3,0.017297600209712983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,7,0.016982400417327882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,2,128,1,float16,float16,8191,0.18605120182037355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,7,0.018534399569034576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,15,0.01903519928455353
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,15,0.016782400012016297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,31,0.01847680062055588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,31,0.017033599317073822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,63,0.018475200235843658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,63,0.017339199781417847
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,127,0.017375999689102174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,127,0.018321600556373597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,255,0.02154400050640106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,255,0.022316800057888032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,511,0.03774079978466034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,511,0.030449599027633667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,1023,0.06223840117454529
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,1023,0.04585599899291992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,2047,0.1042896032333374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,2047,0.07013599872589112
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,4095,0.12025279998779297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,4095,0.1864832043647766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,1,0.02428639978170395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,1,0.027182400226593018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,3,0.024281600117683412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,7,0.024240000545978545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,3,0.027412799000740052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,7,0.027020800113677978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,15,0.024352000653743745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,fp8,8191,0.21620800495147705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,15,0.027219200134277345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,4,128,1,float16,float16,8191,0.3865472078323364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,31,0.0242576003074646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,31,0.027107200026512145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,63,0.024433599412441255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,127,0.025727999210357667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,63,0.026787200570106508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,255,0.04339199960231781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,255,0.035497599840164186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,127,0.031779199838638306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,511,0.06316800117492676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,511,0.052323198318481444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,1023,0.10729279518127441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,1023,0.07825120091438294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,2047,0.1661344051361084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,2047,0.1897312045097351
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,1,0.014884799718856812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,4095,0.2239759922027588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,1,0.011956799775362015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,4095,0.3558448076248169
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,3,0.015241600573062897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,7,0.011286400258541107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,7,0.016312000155448914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,3,0.012129600346088409
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,15,0.012129600346088409
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,15,0.011428800225257874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,31,0.01536639928817749
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,31,0.012174399942159653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,63,0.011321599781513213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,63,0.012263999879360199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,127,0.014185599982738495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,127,0.012135999649763108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,255,0.013158400356769562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,fp8,8191,0.4213856220245361
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,255,0.014236800372600555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,511,0.02028159946203232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,511,0.01803999990224838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,32,8,128,1,float16,float16,8191,0.6896575927734375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,1023,0.015571199357509613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,2047,0.015846399962902068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,2047,0.021270400285720824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,4095,0.01661760061979294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,1023,0.016697600483894348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,float16,8191,0.01834239959716797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,8191,0.018500800430774688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,1,0.014161600172519684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,1,0.01202400028705597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,3,0.011383999884128571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,3,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,1,128,1,float16,fp8,4095,0.017233599722385407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,7,0.01127839982509613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,7,0.012104000151157378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,15,0.011473599821329117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,31,0.011744000017642975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,31,0.012118399888277055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,15,0.011990399658679962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,63,0.011553599685430526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,63,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,127,0.011718399822711945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,127,0.012142399698495865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,255,0.012836800515651703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,255,0.014206400513648987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,511,0.016128000617027283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,511,0.018467199802398682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,1023,0.015732799470424653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,1023,0.01669120043516159
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,2047,0.016171200573444365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,2047,0.017107200622558594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,4095,0.017027199268341064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,4095,0.01732800006866455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,float16,8191,0.01796640008687973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,2,128,1,float16,fp8,8191,0.01879359930753708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,1,0.011428800225257874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,1,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,3,0.011340799927711486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,3,0.012068799883127212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,7,0.011561600118875503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,7,0.012191999703645706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,15,0.011683200299739838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,15,0.01218239963054657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,31,0.011396799981594086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,31,0.01231520026922226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,63,0.011406400054693223
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,63,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,127,0.01183359995484352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,127,0.012204799801111221
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,255,0.013220800459384919
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,255,0.014100800454616546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,511,0.016438399255275727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,511,0.01794400066137314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,1023,0.015532800555229187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,1023,0.016948799788951873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,2047,0.015940800309181213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,2047,0.017272000014781953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,4095,0.016763199865818024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,float16,8191,0.01875839978456497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,4095,0.01759359985589981
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,4,128,1,float16,fp8,8191,0.019017599523067474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,1,0.01143840029835701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,1,0.012516799569129943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,3,0.011420799791812897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,3,0.012068799883127212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,7,0.011649599671363831
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,15,0.011468800157308579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,7,0.012118399888277055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,15,0.012084800004959106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,31,0.0117296002805233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,31,0.012118399888277055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,63,0.011720000207424164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,63,0.012142399698495865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,127,0.011423999816179276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,127,0.012214399874210358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,255,0.013355199992656708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,255,0.01404159963130951
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,511,0.01615840047597885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,511,0.01772959977388382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,1023,0.015611200034618378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,1023,0.016766400635242464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,2047,0.016475200653076172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,2047,0.016921600699424742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,4095,0.01823520064353943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,4095,0.01788959950208664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,float16,8191,0.021831999719142913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,1,0.011655999720096588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,32,8,128,1,float16,fp8,8191,0.02181279957294464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,1,0.012067200243473053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,3,0.011646399646997452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,3,0.01223519966006279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,7,0.011555200070142746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,7,0.012337599694728852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,15,0.011284799873828888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,15,0.01223199963569641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,31,0.011670400202274323
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,31,0.012211199849843979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,63,0.011436799913644791
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,63,0.012385600060224534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,127,0.011321599781513213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,127,0.012289600074291229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,255,0.013209599256515502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,255,0.01419679969549179
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,511,0.016087999939918517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,511,0.0177824005484581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,1023,0.015775999426841734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,1023,0.016707199811935424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,2047,0.016228799521923066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,2047,0.017127999663352968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,4095,0.01701440066099167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,4095,0.017392000555992125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,float16,8191,0.018467199802398682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,1,0.011535999923944473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,1,128,1,float16,fp8,8191,0.018764799833297728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,1,0.01218400001525879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,3,0.011270400136709213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,7,0.011427199840545655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,3,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,7,0.012289600074291229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,15,0.01149279996752739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,15,0.01212640032172203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,31,0.011528000235557556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,31,0.01276959925889969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,63,0.01154239997267723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,63,0.012275200337171555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,127,0.011348800361156463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,127,0.012200000137090683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,255,0.013344000279903411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,255,0.014129599928855896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,511,0.01616320013999939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,511,0.01778720021247864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,1023,0.01584160029888153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,1023,0.01716320067644119
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,2047,0.015911999344825744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,2047,0.01722240000963211
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,4095,0.01781439930200577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,4095,0.016971200704574585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,fp8,8191,0.01871200054883957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,2,128,1,float16,float16,8191,0.019366399943828584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,1,0.011574400216341018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,1,0.012251199781894683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,3,0.011249600350856781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,3,0.01225920021533966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,7,0.01175519973039627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,7,0.012148799747228623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,15,0.011407999694347382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,15,0.012275200337171555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,31,0.011427199840545655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,31,0.012489599734544754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,63,0.011572799831628799
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,63,0.012151999771595002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,127,0.01154239997267723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,127,0.012137600034475327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,255,0.014254400134086609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,255,0.013465599715709686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,511,0.016208000481128693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,511,0.017847999930381775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,1023,0.015647999942302704
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,2047,0.01663520038127899
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,1023,0.016993600130081176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,2047,0.017000000178813934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,4095,0.0182559996843338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,4095,0.017763200402259826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,1,0.011779200285673141
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,float16,8191,0.021904000639915468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,4,128,1,float16,fp8,8191,0.02173600047826767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,1,0.012326399981975555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,3,0.011468800157308579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,3,0.012336000055074691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,7,0.011860799789428712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,15,0.011539199948310852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,7,0.012385600060224534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,15,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,31,0.0117807999253273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,31,0.012595200538635254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,63,0.011502400040626526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,63,0.012302400171756744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,127,0.011644800007343293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,127,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,255,0.013833600282669067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,255,0.014248000085353851
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,511,0.016249600052833556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,511,0.01791359931230545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,1023,0.016257600486278535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,1023,0.017187200486660004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,2047,0.017772799730300902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,2047,0.017427200078964235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,4095,0.021667200326919555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,4095,0.0212336003780365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,float16,8191,0.03725599944591522
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,1,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,32,8,128,1,float16,fp8,8191,0.027478399872779845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,1,0.013417600095272065
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,3,0.012785600125789642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,3,0.013359999656677246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,7,0.012374400347471236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,15,0.012787200510501862
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,7,0.0133775994181633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,15,0.01345279961824417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,31,0.012583999335765839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,31,0.01313599944114685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,63,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,63,0.013631999492645264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,127,0.012579199671745301
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,127,0.013489599525928497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,255,0.01485760062932968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,255,0.014975999295711518
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,511,0.01733279973268509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,511,0.018995200097560883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,1023,0.018729600310325622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,1023,0.01826079934835434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,2047,0.021836799383163453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,2047,0.021480000019073485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,4095,0.03877600133419037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,4095,0.028288000822067262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,1,0.012673600018024445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,float16,8191,0.06027200222015381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,1,0.0135343998670578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,3,0.012718400359153748
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,3,0.013407999277114868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,1,128,1,float16,fp8,8191,0.043166399002075195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,7,0.013451200723648072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,7,0.012656000256538392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,15,0.012619200348854064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,15,0.013387200236320496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,31,0.012875199317932129
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,31,0.013358399271965027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,63,0.012811200320720672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,63,0.01343040019273758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,127,0.012857599556446076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,127,0.01356479972600937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,255,0.014660799503326416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,255,0.015540799498558045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,511,0.017635199427604675
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,511,0.018982400000095368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,1023,0.021033599972724915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,1023,0.021771200001239777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,2047,0.03858399987220764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,2047,0.027952000498771667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,4095,0.05968959927558899
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,4095,0.0421424001455307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,1,0.012673600018024445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,1,0.013478399813175201
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,float16,8191,0.1028704047203064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,3,0.012691199779510498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,2,128,1,float16,fp8,8191,0.06784639954566955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,7,0.012703999876976013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,3,0.01364160031080246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,15,0.012643200159072877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,7,0.013625599443912506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,15,0.013463999330997466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,31,0.012777599692344665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,31,0.013558399677276612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,63,0.01268640011548996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,127,0.012971200048923492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,63,0.013687999546527862
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,127,0.013502399623394012
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,255,0.014838400483131408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,255,0.015473599731922149
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,511,0.017983999848365784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,511,0.019270400702953338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,1023,0.036671999096870425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,1023,0.02712000012397766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,2047,0.05814560055732727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,2047,0.041407999396324155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,4095,0.10057439804077148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,4095,0.06585119962692261
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,1,0.01653439998626709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,1,0.01774719953536987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,3,0.01777600049972534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,3,0.01669439971446991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,fp8,8191,0.11673599481582642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,4,128,1,float16,float16,8191,0.18541120290756224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,7,0.016624000668525696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,7,0.017849600315093993
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,15,0.01672479957342148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,31,0.016569599509239197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,15,0.017921599745750427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,63,0.01659359931945801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,31,0.018033599853515624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,63,0.017953599989414214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,127,0.01685439944267273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,127,0.01815200001001358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,255,0.021828800439834595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,255,0.0208079993724823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,511,0.029356798529624938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,511,0.03669439852237701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,1023,0.05960000157356262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,1023,0.04560959935188293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,2047,0.06940320134162903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,2047,0.10263839960098267
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,1,0.011385600268840789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,4095,0.18563679456710816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,1,0.012217599898576736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,3,0.0114656001329422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,4095,0.11954720020294189
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,7,0.011585599929094314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,7,0.012257599830627441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,3,0.012289600074291229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,15,0.012390399724245072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,fp8,8191,0.2170975923538208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,15,0.011673600226640702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,31,0.011419200152158738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,31,0.012444800138473511
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,32,8,128,1,float16,float16,8191,0.3538448095321655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,63,0.0115167997777462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,127,0.011772800236940384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,127,0.012307199835777282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,255,0.013172799348831176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,63,0.012292800098657608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,511,0.016230399906635284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,511,0.01842560023069382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,1023,0.015862399339675905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,255,0.014697599411010741
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,1023,0.016891199350357055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,2047,0.016139200329780577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,2047,0.01748960018157959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,4095,0.016956800222396852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,4095,0.018475200235843658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,float16,8191,0.019091199338436126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,1,128,1,float16,fp8,8191,0.01902720034122467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,1,0.01173280030488968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,1,0.012241599708795547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,3,0.01204639971256256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,3,0.012300799787044524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,7,0.01154239997267723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,7,0.01260959953069687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,15,0.011407999694347382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,15,0.014319999516010285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,31,0.011457599699497223
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,31,0.012270399928092956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,63,0.011740799993276596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,63,0.012164799869060517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,127,0.013414399325847625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,127,0.012281599640846252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,255,0.013240000605583191
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,255,0.014382399618625641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,511,0.01603520065546036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,511,0.02109919935464859
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,1023,0.016950400173664094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,1023,0.015820799767971037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,2047,0.01658560037612915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,2047,0.017182399332523347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,4095,0.0178384006023407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,4095,0.021396799385547637
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,float16,8191,0.022342400252819063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,1,0.011723200231790543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,2,128,1,float16,fp8,8191,0.022049599885940553
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,1,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,3,0.013575999438762665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,7,0.011727999895811081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,15,0.011723200231790543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,3,0.012299200147390365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,7,0.012256000190973282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,15,0.013491199910640716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,31,0.011868800222873687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,31,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,63,0.012622399628162384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,63,0.012382400035858155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,127,0.011472000181674958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,127,0.012460800260305405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,255,0.013465599715709686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,255,0.014185599982738495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,511,0.016089600324630738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,511,0.019659200310707094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,1023,0.016152000427246092
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,1023,0.016926400363445282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,2047,0.01783200055360794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,2047,0.01738400012254715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,4095,0.023012800514698027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,4095,0.021403199434280394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,1,0.011771199852228164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,float16,8191,0.03742560148239136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,4,128,1,float16,fp8,8191,0.02744159996509552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,1,0.013407999277114868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,3,0.011854399740695954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,3,0.012451200187206269
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,7,0.011844799667596818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,15,0.012169600278139115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,15,0.012891200184822083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,31,0.011854399740695954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,7,0.012720000743865967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,31,0.012563200294971466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,63,0.011767999827861786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,63,0.013096000254154205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,127,0.011844799667596818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,127,0.012904000282287598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,255,0.013612799346446991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,255,0.014620800316333771
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,511,0.016728000342845918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,511,0.018755200505256652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,1023,0.017641599476337432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,1023,0.01723040044307709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,2047,0.0196943998336792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,2047,0.021135999262332915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,4095,0.028259199857711793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,4095,0.03681919872760773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,float16,8191,0.05865280032157898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,1,0.014534400403499603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,1,0.015332800149917603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,32,8,128,1,float16,fp8,8191,0.041703999042510986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,3,0.01568640023469925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,3,0.015516799688339234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,7,0.015384000539779664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,7,0.014646400511264802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,15,0.01451839953660965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,15,0.01581760048866272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,31,0.014983999729156493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,63,0.015828800201416016
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,31,0.015518400073051452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,63,0.014803199470043183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,127,0.014694400131702423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,127,0.015643200278282164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,255,0.017683200538158417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,255,0.017316800355911256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,511,0.020078399777412416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,511,0.02099040001630783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,1023,0.030115199089050294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,1023,0.039238399267196654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,2047,0.06006720066070557
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,2047,0.0429967999458313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,1,0.018563200533390046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,1,0.01979839950799942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,fp8,4095,0.0700655996799469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,1,128,1,float16,float16,4095,0.1035040020942688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,3,0.019916799664497376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,3,0.018320000171661376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,7,0.018062399327754976
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,15,0.02003040015697479
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,31,0.01828159987926483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,7,0.02045920044183731
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,15,0.018063999712467194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,31,0.01971839964389801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,63,0.019064000248908995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,63,0.0200095996260643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,127,0.018334400653839112
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,127,0.019655999541282655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,255,0.022286400198936462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,255,0.024633599817752837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,511,0.040633600950241086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,511,0.03208160102367401
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,1023,0.06412320137023926
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,1023,0.04777120053768158
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,2047,0.07239840030670167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,2047,0.10631999969482422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,1,0.02584159970283508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,1,0.028515198826789857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,3,0.02868959903717041
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,fp8,4095,0.12238080501556396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,3,0.02641119956970215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,7,0.025684800744056702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,7,0.02850080132484436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,2,128,1,float16,float16,4095,0.1890544056892395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,15,0.025843200087547303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,31,0.026318401098251343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,31,0.028699201345443726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,63,0.02563199996948242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,15,0.028892800211906433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,127,0.026824000477790832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,63,0.029080000519752503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,255,0.04506239891052246
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,127,0.02942720055580139
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,255,0.03765920102596283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,511,0.06630399823188782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,511,0.05320799946784973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,1023,0.07957440018653869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,1023,0.11033120155334472
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,2047,0.19240479469299315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,1,0.0399728000164032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,2047,0.12747360467910768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,1,0.04654879868030548
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,3,0.04605759978294373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,3,0.041073599457740785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,7,0.03994719982147217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,7,0.04654879868030548
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,15,0.03991200029850006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,15,0.047182399034500125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,float16,4095,0.3569952011108398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,4,128,1,float16,fp8,4095,0.22655680179595947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,31,0.03996959924697876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,31,0.0465503990650177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,63,0.055726397037506106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,63,0.04192320108413696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,127,0.051560002565383914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,255,0.07329919934272766
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,127,0.04930559992790222
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,255,0.06663359999656678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,511,0.1173967957496643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,511,0.0944320023059845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,1023,0.1437872052192688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,1023,0.1967568039894104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,2047,0.24089601039886474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,2047,0.36489119529724123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,1,0.020633600652217865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,1,0.022313599288463593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,3,0.02107519954442978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,7,0.02067359983921051
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,7,0.022198399901390074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,15,0.021084800362586975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,3,0.02232159972190857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,15,0.022302399575710296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,fp8,4095,0.4343696117401123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,31,0.020820799469947814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,63,0.021089600026607515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,31,0.0221903994679451
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,63,0.0221903994679451
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,32,8,128,1,float16,float16,4095,0.7009696006774903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,127,0.022273600101470947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,255,0.025455999374389648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,127,0.020721599459648132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,255,0.026212799549102783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,511,0.043670400977134705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,float16,1023,0.06858559846878051
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,1023,0.050551998615264895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,1,0.02826080024242401
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,1,128,1,float16,fp8,511,0.060318398475646975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,1,0.030817601084709167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,3,0.027956798672676086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,7,0.02802720069885254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,15,0.02799200117588043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,3,0.03097760081291199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,7,0.05365440249443054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,15,0.030822399258613586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,31,0.02794240117073059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,31,0.03120799958705902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,63,0.0280784010887146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,63,0.031564798951148984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,127,0.047040000557899475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,127,0.03139359951019287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,255,0.047774401307106015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,255,0.04018079936504364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,511,0.0686623990535736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,511,0.05629760026931763
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,1,0.04880320131778717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,1,0.04271839857101441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,3,0.04258399903774261
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,fp8,1023,0.08244799971580505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,3,0.048809599876403806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,2,128,1,float16,float16,1023,0.13280160427093507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,7,0.042715200781822206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,7,0.04881280064582825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,15,0.04262399971485138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,15,0.04875200092792511
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,31,0.04289759993553162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,31,0.07163199782371521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,63,0.04891839921474457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,63,0.04457600116729736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,127,0.05191199779510498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,127,0.05081920027732849
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,255,0.07434399724006653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,255,0.09219359755516052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,511,0.11602239608764649
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,511,0.09647200107574463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,1,0.07196000218391418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,fp8,1023,0.14740320444107055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,1,0.11707040071487426
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,3,0.0704367995262146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,3,0.08353599905967712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,4,128,1,float16,float16,1023,0.20370879173278808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,7,0.07200639843940734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,7,0.08363999724388123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,15,0.08884639739990234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,31,0.07092480063438415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,15,0.08343520164489746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,31,0.08367840051651002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,63,0.07807520031929016
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,63,0.08622239828109741
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,127,0.09866560101509095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,127,0.09578080177307129
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,255,0.12423839569091796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,255,0.118777596950531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,1,0.03419359922409058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,1,0.036977601051330564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,511,0.17726559638977052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,511,0.21385600566864013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,3,0.03404319882392883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,3,0.03735199868679047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,7,0.0343311995267868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,7,0.037187200784683225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,15,0.03461759984493255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,15,0.03723680078983307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,31,0.03419359922409058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,31,0.03744640052318573
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,fp8,1023,0.27671198844909667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,63,0.03423199951648712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,32,8,128,1,float16,float16,1023,0.38114399909973146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,63,0.03715679943561554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,127,0.038545599579811095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,127,0.03766080141067505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,255,0.05370240211486817
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,1,0.04819200038909912
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,255,0.0460783988237381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,fp8,511,0.06244480013847351
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,1,0.05418720245361328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,1,128,1,float16,float16,511,0.08003680109977722
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,3,0.04905279874801636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,3,0.05460799932479858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,7,0.04819679856300354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,7,0.053932797908782956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,15,0.04840799868106842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,15,0.054155200719833374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,31,0.06421279907226562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,31,0.055022400617599485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,63,0.05421440005302429
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,63,0.05145440101623535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,127,0.058433598279953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,127,0.056531202793121335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,255,0.09445279836654663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,255,0.0821008026599884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,1,0.07709599733352661
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,float16,511,0.12422239780426025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,1,0.0886784017086029
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,3,0.07652959823608399
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,2,128,1,float16,fp8,511,0.10253759622573852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,3,0.11904959678649903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,7,0.07710559964179993
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,15,0.07597119808197021
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,15,0.08901119828224183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,7,0.09012159705162048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,31,0.07620319724082947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,31,0.11099679470062256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,63,0.09318240284919739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,63,0.08352159857749938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,127,0.08711519837379456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,127,0.09834079742431641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,255,0.12894400358200073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,255,0.15193439722061158
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,1,0.13171199560165406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,float16,511,0.21519041061401367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,1,0.15613600015640258
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,4,128,1,float16,fp8,511,0.18315999507904052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,3,0.12931360006332399
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,3,0.19748159646987914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,7,0.1318560004234314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,15,0.1297152042388916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,7,0.1555456042289734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,31,0.13733439445495604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,15,0.15732799768447875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,63,0.14262559413909912
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,31,0.1990288019180298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,63,0.1672271966934204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,127,0.1490015983581543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,127,0.17749600410461425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,1,0.01170239970088005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,255,0.23058559894561767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,3,0.011662399768829346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,1,0.012283200025558471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,255,0.2252000093460083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,3,0.012558400630950928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,7,0.01162080019712448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,7,0.01228799968957901
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,15,0.011736000329256058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,15,0.012412799894809723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,31,0.011803200095891952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,float16,511,0.4239984035491943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,32,8,128,1,float16,fp8,511,0.3416624069213867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,31,0.012276799976825714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,63,0.011767999827861786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,63,0.012375999987125397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,127,0.011822400242090225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,255,0.013228799402713775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,511,0.016531200706958772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,511,0.01794559955596924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,1023,0.01627359986305237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,127,0.01433439999818802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,255,0.014396800100803376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,1023,0.017080000042915343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,2047,0.01672479957342148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,2047,0.017374399304389953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,4095,0.018566399812698364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,4095,0.024879999458789825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,1,0.012060800194740295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,float16,8191,0.022753599286079406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,1,0.01255200058221817
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,1,128,1,float16,fp8,8191,0.021998399496078493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,3,0.01167680025100708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,7,0.012060800194740295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,3,0.017217600345611574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,7,0.01252480000257492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,15,0.011856000125408172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,15,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,31,0.011843200027942657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,31,0.01276639997959137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,63,0.01619359999895096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,63,0.012392000108957291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,127,0.011664000153541566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,127,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,255,0.014796799421310425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,511,0.022711999714374542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,255,0.013468800485134125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,511,0.01781280040740967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,1023,0.016139200329780577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,1023,0.017118400335311888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,2047,0.018385599553585052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,2047,0.024465599656105043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,4095,0.02173759937286377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,4095,0.02113119959831238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,float16,8191,0.03780960142612457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,1,0.012195199728012085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,1,0.01733279973268509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,2,128,1,float16,fp8,8191,0.02773439884185791
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,3,0.01194560006260872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,3,0.012361600250005721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,7,0.011801599711179733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,7,0.012800000607967377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,15,0.0141744002699852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,15,0.0126351997256279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,31,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,31,0.0119439996778965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,63,0.015036800503730774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,63,0.011838400363922119
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,127,0.01207519993185997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,127,0.01276479959487915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,255,0.013868799805641175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,255,0.017140799760818483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,511,0.01682240068912506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,511,0.018401600420475006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,1023,0.017263999581336974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,1023,0.017696000635623932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,2047,0.023391999304294586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,2047,0.021096000075340272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,4095,0.026825600862503053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,4095,0.03761279881000519
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,float16,8191,0.05903519988059998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,1,0.012039999663829803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,4,128,1,float16,fp8,8191,0.044516798853874204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,1,0.012622399628162384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,3,0.012191999703645706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,3,0.012759999930858612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,7,0.012041600048542022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,15,0.011913599818944931
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,15,0.01292479932308197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,7,0.013819199800491334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,31,0.012089599668979645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,31,0.01286720037460327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,63,0.011907199770212174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,63,0.012904000282287598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,127,0.013038399815559387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,127,0.012870399653911591
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,255,0.014032000303268432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,255,0.014654399454593658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,511,0.017089599370956422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,511,0.019864000380039215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,1023,0.01983039975166321
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,1023,0.020975999534130096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,2047,0.03652639985084534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,2047,0.02802880108356476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,4095,0.059571200609207155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,4095,0.042552000284194945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,float16,8191,0.10060319900512696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,1,0.05523520112037659
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,32,8,128,1,float16,fp8,8191,0.06592159867286682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,1,0.0631376028060913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,3,0.0547760009765625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,3,0.06035040020942688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,7,0.055315202474594115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,15,0.05674399733543396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,7,0.060164797306060794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,15,0.06064159870147705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,31,0.05467519760131836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,63,0.06383360028266907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,31,0.0614736020565033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,63,0.06087679862976074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,127,0.06433759927749634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,127,0.06594240069389343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,float16,255,0.08646399974822998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,1,0.08286399841308593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,1,0.09522719979286194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,1,128,1,float16,fp8,255,0.08054239749908447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,7,0.08231840133666993
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,3,0.08322399854660034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,3,0.09362720251083374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,7,0.09395999908447265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,15,0.08274080157279969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,31,0.08471360206604003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,15,0.09548159837722778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,31,0.09430559873580932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,63,0.08888959884643555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,63,0.10068639516830444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,127,0.09588479995727539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,127,0.10350240468978882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,fp8,255,0.12923200130462648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,2,128,1,float16,float16,255,0.13810080289840698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,1,0.1371343970298767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,1,0.16166399717330932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,3,0.13837759494781493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,7,0.13750720024108887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,3,0.1610640048980713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,15,0.1390544056892395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,7,0.17322720289230348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,31,0.1442911982536316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,15,0.16124160289764405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,63,0.14677280187606812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,31,0.16590399742126466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,63,0.17115999460220338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,127,0.15592479705810547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,127,0.1878224015235901
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,float16,255,0.22841598987579345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,1,0.24261600971221925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,4,128,1,float16,fp8,255,0.22668960094451904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,3,0.24223520755767822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,1,0.2927344083786011
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,3,0.2936511993408203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,7,0.29213600158691405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,15,0.25570878982543943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,7,0.2915136098861694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,31,0.2573359966278076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,15,0.2933120012283325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,63,0.2633791923522949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,31,0.31493439674377444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,63,0.38524959087371824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,1,0.09686880111694336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,127,0.2809231996536255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,127,0.3282032012939453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,1,0.1334223985671997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,3,0.09653599858283997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,3,0.10688639879226684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,7,0.09559040069580078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,float16,255,0.41981282234191897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,32,8,128,1,float16,fp8,255,0.4216479778289795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,7,0.1335376024246216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,15,0.0984112024307251
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,15,0.10819519758224487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,31,0.10042400360107422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,31,0.11007519960403442
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,63,0.11464639902114868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,63,0.12193280458450317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,float16,127,0.11221760511398315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,1,128,1,float16,fp8,127,0.11730879545211792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,1,0.15100640058517456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,1,0.172817599773407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,3,0.18144160509109497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,3,0.1718943953514099
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,7,0.1530784010887146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,7,0.1730847954750061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,15,0.15321600437164307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,15,0.17494399547576905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,31,0.18418400287628173
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,31,0.18058079481124878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,63,0.18432639837265014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,63,0.16123199462890625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,float16,127,0.16958080530166625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,2,128,1,float16,fp8,127,0.18698079586029054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,1,0.30789120197296144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,3,0.26343679428100586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,1,0.30793440341949463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,3,0.30558080673217775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,7,0.2636751890182495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,7,0.3073920011520386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,31,0.2677599906921387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,15,0.31423680782318114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,15,0.3164560079574585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,31,0.320417594909668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,63,0.2720207929611206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,63,0.3236128091812134
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,float16,127,0.32849760055541993
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,4,128,1,float16,fp8,127,0.32865281105041505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,1,0.48874402046203613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,3,0.48639678955078125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,1,0.5717711925506592
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,3,0.573576021194458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,15,0.4905375957489014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,7,0.5824463844299317
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,7,0.6624767780303955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,31,0.4946256160736084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,15,0.5937727928161621
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,1,0.011528000235557556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,31,0.5967535972595215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,1,0.01231200024485588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,3,0.012276799976825714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,3,0.01146399974822998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,63,0.500278377532959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,7,0.011500799655914306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,7,0.012225600332021714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,15,0.01223519966006279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,63,0.6003039836883545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,31,0.011550399661064147
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,15,0.011777599900960922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,31,0.01231200024485588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,63,0.011532799899578094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,127,0.011710400134325028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,63,0.012307199835777282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,float16,127,0.6946479797363281
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,127,0.012368000298738479
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,255,0.013555200397968292
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,255,0.01440960019826889
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,32,8,128,1,float16,fp8,127,0.6191440105438233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,511,0.016463999450206757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,1023,0.01622239947319031
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,1023,0.01685280054807663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,2047,0.017824000120162962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,511,0.026612800359725953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,4095,0.021929599344730377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,2047,0.01788959950208664
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,4095,0.021211199462413788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,1,0.01677599996328354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,fp8,8191,0.02741599977016449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,1,128,1,float16,float16,8191,0.03855839967727661
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,1,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,3,0.011935999989509583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,7,0.016756799817085267
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,3,0.01239679977297783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,7,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,15,0.011964800208806992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,15,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,31,0.01199359968304634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,31,0.01558080017566681
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,63,0.011950399726629257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,63,0.012907199561595917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,127,0.011822400242090225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,127,0.012777599692344665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,255,0.014102399349212646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,255,0.018518400192260743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,511,0.016680000722408293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,511,0.0180976003408432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,1023,0.017505599558353423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,1023,0.017233599722385407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,2047,0.024553599953651428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,2047,0.02091519981622696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,4095,0.03753440082073212
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,4095,0.0270687997341156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,1,0.011956799775362015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,float16,8191,0.058950400352478026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,1,0.012638400495052337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,3,0.012014400213956833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,2,128,1,float16,fp8,8191,0.050444799661636355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,3,0.012595200538635254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,7,0.0117807999253273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,7,0.012680000066757202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,15,0.01210559979081154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,31,0.012014400213956833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,15,0.01443679928779602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,31,0.012591999769210816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,63,0.011939200013875962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,127,0.013582399487495423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,63,0.012721599638462066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,127,0.012702399492263794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,255,0.013790400326251983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,255,0.01470080018043518
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,511,0.018667200207710268
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,511,0.01804479956626892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,1023,0.01947840005159378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,1023,0.02043839991092682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,2047,0.03025600016117096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,2047,0.037278398871421814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,4095,0.05918400287628174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,4095,0.04112800061702728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,1,0.013513599336147309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,1,0.012863999605178833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,3,0.011948800086975098
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,3,0.012705600261688233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,fp8,8191,0.0676639974117279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,7,0.013369600474834441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,7,0.012839999794960023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,4,128,1,float16,float16,8191,0.10126240253448486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,15,0.011843200027942657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,15,0.012664000689983367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,31,0.012960000336170197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,31,0.012788799405097962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,63,0.012203200161457062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,63,0.012849600613117218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,127,0.012787200510501862
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,127,0.012204799801111221
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,255,0.01491519957780838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,255,0.014865599572658539
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,511,0.017587199807167053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,511,0.018308800458908082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,1023,0.035872000455856326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,1023,0.02587519884109497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,2047,0.05792800188064575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,2047,0.04057759940624237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,4095,0.06471359729766846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,4095,0.09965119957923889
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,1,0.013944000005722046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,1,0.013657599687576294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,7,0.013689599931240082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,7,0.014188799262046813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,3,0.013388800621032714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,15,0.013051199913024902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,3,0.014132800698280334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,15,0.014375999569892883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,float16,8191,0.18285919427871705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,32,8,128,1,float16,fp8,8191,0.11388159990310669
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,31,0.013420799374580383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,31,0.013889600336551667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,63,0.013171200454235078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,63,0.014617599546909332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,127,0.013371199369430542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,127,0.013872000575065612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,255,0.01640480011701584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,255,0.01475680023431778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,511,0.017679999768733978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,511,0.019411200284957887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,1023,0.021657599508762358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,1023,0.022249600291252135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,2047,0.03825600147247314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,2047,0.028763198852539064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,4095,0.060499197244644164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,4095,0.04187679886817932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,8191,0.10306240320205688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,8191,0.06767839789390565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,1,0.013542400300502777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,1,0.014004799723625182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,3,0.01334560066461563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,fp8,16383,0.11615040302276611
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,1,128,1,float16,float16,16383,0.18816800117492677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,7,0.013633599877357483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,3,0.014140799641609192
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,7,0.01422239989042282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,15,0.013350400328636169
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,15,0.014023999869823455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,31,0.01308320015668869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,31,0.014504000544548035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,63,0.013875199854373932
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,127,0.013808000087738036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,63,0.013660800457000733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,255,0.015174399316310882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,127,0.01412159949541092
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,255,0.016518400609493257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,511,0.01852799952030182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,1023,0.02672959864139557
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,511,0.019870400428771973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,1023,0.03763200044631958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,2047,0.058963197469711306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,2047,0.04171999990940094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,4095,0.10137120485305787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,4095,0.06534079909324646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,8191,0.11660959720611572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,8191,0.1859007954597473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,1,0.01871519982814789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,1,0.016708800196647645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,7,0.0173007994890213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,3,0.017395199835300447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,7,0.01830720007419586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,3,0.018636800348758698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,fp8,16383,0.23709280490875245
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,15,0.017377600073814392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,15,0.01881600022315979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,31,0.0168272003531456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,2,128,1,float16,float16,16383,0.3554800033569336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,31,0.01860959976911545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,63,0.018873600661754607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,63,0.030265599489212036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,127,0.01751199960708618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,127,0.01858240067958832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,255,0.022643199563026427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,511,0.0369951993227005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,255,0.02134400010108948
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,511,0.03069919943809509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,1023,0.06206079721450806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,1023,0.04585280120372772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,2047,0.10394079685211181
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,2047,0.07099360227584839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,4095,0.12052639722824096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,4095,0.18801759481430053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,1,0.02468000054359436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,1,0.05431680083274841
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,8191,0.2179647922515869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,3,0.024663999676704407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,8191,0.3535712003707886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,3,0.02980000078678131
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,7,0.027584001421928406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,7,0.024163199961185454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,15,0.02987839877605438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,15,0.024475200474262236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,31,0.02508159875869751
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,31,0.027214398980140685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,63,0.02725600004196167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,fp8,16383,0.4160128116607666
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,63,0.02755360007286072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,127,0.025758400559425354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,127,0.027233600616455078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,255,0.044326400756835936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,4,128,1,float16,float16,16383,0.68930082321167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,511,0.06244639754295349
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,255,0.03629119992256165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,511,0.0530784010887146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,1023,0.10659680366516114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,1023,0.08608800172805786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,2047,0.20079519748687744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,2047,0.1255728006362915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,4095,0.2265712022781372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,4095,0.3549407958984375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,3,0.011552000045776367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,1,0.011444800347089768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,1,0.012387199699878693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,3,0.01223680004477501
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,7,0.01154400035738945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,7,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,15,0.011582399904727935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,15,0.012352000176906585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,31,0.01234079971909523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,31,0.01165279969573021
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,63,0.011579199880361556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,8191,0.6899936199188232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,63,0.012265600264072418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,127,0.01162559986114502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,127,0.01231039986014366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,255,0.014251199364662171
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,511,0.016124799847602844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,511,0.018398399651050567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,8191,0.8979743957519531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,1023,0.015507200360298156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,255,0.013027200102806091
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,1023,0.016752000153064727
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,2047,0.016094399988651274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,4095,0.01642400026321411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,8191,0.01804320067167282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,2047,0.025470399856567384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,8191,0.018780800700187682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,4095,0.017215999960899352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,float16,16383,0.020446400344371795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,1,128,1,float16,fp8,16383,0.020710399746894835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,1,0.011320000141859054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,fp8,16383,0.8022959709167481
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,1,0.012459199875593185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,3,0.011622399836778641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,3,0.012371200323104858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,7,0.011367999762296677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,15,0.011675199866294861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,15,0.012555199861526489
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,31,0.011604800075292587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,31,0.012092799693346024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,63,0.011667200177907944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,63,0.012563200294971466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,7,0.012534399330615998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,127,0.011523199826478958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,127,0.01226240023970604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,255,0.013145600259304047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,255,0.014454400539398194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,511,0.018193599581718446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,16,8,128,1,float16,float16,16383,1.3567680358886718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,511,0.016380800306797026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,1023,0.016734400391578676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,1023,0.015390400588512421
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,2047,0.01629440039396286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,2047,0.017089599370956422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,4095,0.017867200076580048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,8191,0.01828159987926483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,8191,0.018606400489807128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,16383,0.02099999934434891
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,fp8,16383,0.0208079993724823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,2,128,1,float16,float16,4095,0.01730719953775406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,1,0.011532799899578094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,1,0.012532800436019897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,3,0.0115167997777462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,3,0.012355200201272964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,7,0.011537600308656693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,15,0.011497599631547928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,15,0.012240000069141388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,7,0.01255040019750595
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,31,0.011534400284290314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,31,0.01249760016798973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,63,0.011604800075292587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,63,0.012249600142240524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,127,0.011628799885511399
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,127,0.012254399806261062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,255,0.013232000172138214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,255,0.014326399564743042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,511,0.017977599799633027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,511,0.016680000722408293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,1023,0.015892800688743592
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,1023,0.016889600455760954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,2047,0.016206400096416475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,2047,0.017316800355911256
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,4095,0.016756799817085267
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,4095,0.01801439970731735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,8191,0.019064000248908995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,8191,0.018811200559139252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,1,0.011579199880361556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,float16,16383,0.022942399978637694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,4,128,1,float16,fp8,16383,0.02305919975042343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,1,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,3,0.011598400026559829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,3,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,7,0.011584000289440155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,7,0.01225920021533966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,15,0.011736000329256058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,15,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,31,0.011590400338172912
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,31,0.012399999797344208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,63,0.011668799817562104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,63,0.012251199781894683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,127,0.011856000125408172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,127,0.012323199957609176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,255,0.013254399597644805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,255,0.014324800670146942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,511,0.016305600106716157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,511,0.0178384006023407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,1023,0.01613759994506836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,1023,0.01698080003261566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,2047,0.01647839993238449
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,2047,0.017113600671291352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,4095,0.018518400192260743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,4095,0.018139199912548067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,8191,0.022393600642681123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,8191,0.021980799734592438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,1,0.01159520000219345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,float16,16383,0.03763520121574402
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,16,8,128,1,float16,fp8,16383,0.02789599895477295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,1,0.012280000001192093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,3,0.011512000113725662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,3,0.012455999851226807
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,7,0.011710400134325028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,15,0.011444800347089768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,7,0.012318400293588638
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,31,0.012278400361537933
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,15,0.01234079971909523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,31,0.01156959980726242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,63,0.01162400022149086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,63,0.012316799908876418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,127,0.011622399836778641
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,127,0.012345600128173827
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,255,0.013278399407863618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,255,0.014440000057220459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,511,0.016209599375724793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,511,0.01841920018196106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,1023,0.015868799388408662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,1023,0.016748799383640288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,2047,0.01592160016298294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,2047,0.01719679981470108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,4095,0.01691199988126755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,4095,0.017652800679206847
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,8191,0.018428799510002137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,8191,0.018913599848747253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,float16,16383,0.020670400559902193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,1,128,1,float16,fp8,16383,0.02152640074491501
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,1,0.011425600200891495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,3,0.011561600118875503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,1,0.012380799651145935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,3,0.012336000055074691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,7,0.011481600254774094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,7,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,15,0.01144160032272339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,15,0.012305600196123123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,31,0.011638399958610535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,31,0.012404800206422806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,63,0.011497599631547928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,127,0.011519999802112579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,63,0.012328000366687774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,127,0.012252800166606903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,255,0.013201600313186646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,255,0.014212800562381745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,511,0.016196799278259278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,511,0.01791519969701767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,1023,0.015964800119400026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,1023,0.017118400335311888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,2047,0.016257600486278535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,2047,0.017124800384044646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,4095,0.01743520051240921
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,4095,0.016913600265979767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,8191,0.019068799912929535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,8191,0.018985599279403687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,float16,16383,0.022944000363349915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,2,128,1,float16,fp8,16383,0.022303999960422517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,1,0.011535999923944473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,1,0.012380799651145935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,3,0.011708799749612808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,3,0.01226240023970604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,7,0.011539199948310852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,7,0.012212800234556198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,15,0.01143840029835701
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,15,0.012299200147390365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,31,0.011720000207424164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,31,0.012332800030708312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,63,0.011640000343322753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,63,0.012227199971675873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,127,0.011503999680280685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,127,0.01233920007944107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,255,0.014259199798107147
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,255,0.013278399407863618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,511,0.01618559956550598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,511,0.017763200402259826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,1023,0.015612800419330598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,1023,0.016939200460910797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,2047,0.016251200437545778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,2047,0.017496000230312347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,4095,0.01807519942522049
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,4095,0.01794559955596924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,8191,0.021489599347114564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,8191,0.02188960015773773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,float16,16383,0.037513598799705505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,1,0.011529599875211715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,4,128,1,float16,fp8,16383,0.028652799129486085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,1,0.01223360002040863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,3,0.011641599982976914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,3,0.012414400279521943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,7,0.011681599915027619
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,7,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,15,0.011584000289440155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,31,0.011708799749612808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,15,0.012656000256538392
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,31,0.01234079971909523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,63,0.011689600348472596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,63,0.012252800166606903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,127,0.012318400293588638
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,127,0.011723200231790543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,255,0.013675199449062347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,255,0.014422400295734406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,511,0.01637919992208481
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,511,0.018030400574207305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,1023,0.01607999950647354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,1023,0.016940799355506898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,2047,0.01804800033569336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,2047,0.01749439984560013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,4095,0.022299200296401978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,4095,0.021164800226688384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,8191,0.03697440028190613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,8191,0.02784320116043091
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,float16,16383,0.058297598361968996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,1,0.012587200105190276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,16,8,128,1,float16,fp8,16383,0.04194239974021911
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,1,0.013387200236320496
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,3,0.012318400293588638
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,7,0.012567999958992004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,15,0.01226240023970604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,7,0.013179199397563934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,15,0.013302400708198547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,3,0.013265599310398103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,31,0.012699200212955475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,31,0.013177600502967835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,63,0.012753599882125854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,63,0.013782399892807006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,127,0.01268800050020218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,127,0.013232000172138214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,255,0.014377599954605103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,255,0.015219199657440185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,511,0.0177279993891716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,511,0.01902880072593689
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,1023,0.018593600392341612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,1023,0.018137599527835845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,2047,0.02083519995212555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,2047,0.02181120067834854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,4095,0.03866080045700073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,4095,0.0289247989654541
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,8191,0.05964959859848022
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,8191,0.04208320081233978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,1,0.012876799702644348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,float16,16383,0.10284960269927979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,1,0.01345600038766861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,1,128,1,float16,fp8,16383,0.0677344024181366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,3,0.01300320029258728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,3,0.013700799643993377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,7,0.013390399515628815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,7,0.012615999579429627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,15,0.012699200212955475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,15,0.013659200072288514
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,31,0.012835200130939483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,31,0.013396799564361572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,63,0.012574400007724761
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,63,0.013526399433612824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,127,0.012892800569534301
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,127,0.013739199936389923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,255,0.014553600549697876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,255,0.015612800419330598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,511,0.017455999553203583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,511,0.019208000600337984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,1023,0.020334400236606598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,1023,0.021566399931907655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,2047,0.03832640051841736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,2047,0.02781279981136322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,4095,0.05940319895744324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,4095,0.042375999689102176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,8191,0.10252959728240967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,8191,0.06832640171051026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,1,0.012772800028324127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,1,0.013366399705410004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,float16,16383,0.18672480583190917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,2,128,1,float16,fp8,16383,0.1193552017211914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,3,0.013662399351596832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,3,0.012775999307632447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,7,0.01279360055923462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,15,0.01356000006198883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,31,0.012878400087356568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,7,0.013804799318313599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,31,0.013513599336147309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,15,0.01297599971294403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,63,0.01284160017967224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,63,0.013558399677276612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,127,0.013064000010490417
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,127,0.013459199666976928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,255,0.015652799606323244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,255,0.014812800288200378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,511,0.018049600720405578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,511,0.01923999935388565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,1023,0.035920000076293944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,1023,0.02645280063152313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,2047,0.0579695999622345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,2047,0.04196160137653351
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,4095,0.06693919897079467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,4095,0.10102560520172119
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,8191,0.1161471962928772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,1,0.01653439998626709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,8191,0.18379039764404298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,1,0.017795200645923614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,3,0.01800000071525574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,3,0.01655679941177368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,7,0.017836800217628478
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,7,0.01643519997596741
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,float16,16383,0.3530224084854126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,4,128,1,float16,fp8,16383,0.21495840549468995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,15,0.016387200355529784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,31,0.01890240013599396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,15,0.017824000120162962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,31,0.01797440052032471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,127,0.01666560024023056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,63,0.016489599645137788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,63,0.018134400248527527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,127,0.02045599967241287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,255,0.020428800582885744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,255,0.02199999988079071
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,511,0.036006399989128114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,511,0.02969599962234497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,1023,0.06222720146179199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,1023,0.045316800475120544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,2047,0.10168479681015015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,2047,0.06960319876670837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,4095,0.1844159960746765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,4095,0.1381824016571045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,1,0.013094399869441987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,1,0.013846400380134582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,8191,0.35276639461517334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,8191,0.21916639804840088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,3,0.01297439932823181
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,7,0.012963199615478515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,15,0.012999999523162841
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,7,0.012352000176906585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,3,0.012243200093507767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,15,0.014396800100803376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,31,0.011582399904727935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,31,0.012283200025558471
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,63,0.013376000523567199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,fp8,16383,0.41212158203125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,16,8,128,1,float16,float16,16383,0.6884175777435303
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,63,0.012353599816560746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,127,0.011484800279140473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,127,0.01432960033416748
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,511,0.016212800145149232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,255,0.013264000415802002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,511,0.01807360053062439
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,1023,0.018755200505256652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,255,0.014228799939155578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,1023,0.016864000260829924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,2047,0.01595039963722229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,2047,0.020556800067424774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,4095,0.017150400578975676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,8191,0.019100800156593323
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,4095,0.01777600049972534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,8191,0.02232320010662079
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,float16,16383,0.022628800570964815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,1,128,1,float16,fp8,16383,0.022812800109386445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,1,0.011472000181674958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,1,0.012467200309038163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,3,0.012401600182056428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,7,0.01157120019197464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,3,0.012247999757528305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,7,0.012355200201272964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,15,0.011987199634313583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,15,0.013193599879741669
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,31,0.01165279969573021
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,31,0.012292800098657608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,63,0.011558400094509124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,63,0.01252480000257492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,127,0.012363199889659882
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,127,0.012348800152540206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,255,0.013246400654315949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,255,0.014243200421333313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,511,0.016302399337291718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,511,0.01932159960269928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,1023,0.015974399447441102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,1023,0.01687040030956268
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,2047,0.016288000345230102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,2047,0.017393599450588226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,4095,0.019540800154209136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,4095,0.01815840005874634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,8191,0.021840000152587892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,8191,0.021715199947357176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,1,0.011764799803495407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,fp8,16383,0.028859201073646545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,2,128,1,float16,float16,16383,0.03840959966182709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,1,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,3,0.011633600294589996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,3,0.01252480000257492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,7,0.012084800004959106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,15,0.0117296002805233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,7,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,31,0.011582399904727935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,15,0.012300799787044524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,31,0.012457600235939026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,63,0.012299200147390365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,63,0.012379200011491776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,127,0.011857599765062333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,127,0.012375999987125397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,255,0.013540799915790557
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,511,0.017041599750518797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,255,0.014459200203418732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,511,0.017929600179195405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,1023,0.016100800037384032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,1023,0.017041599750518797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,2047,0.017752000689506532
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,2047,0.01818400025367737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,4095,0.021704000234603883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,4095,0.02093600034713745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,8191,0.036985599994659425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,8191,0.027959999442100526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,1,0.012088000029325485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,float16,16383,0.058955198526382445
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,1,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,4,128,1,float16,fp8,16383,0.04288319945335388
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,3,0.011903999745845795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,3,0.012702399492263794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,7,0.011907199770212174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,15,0.011761599779129028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,7,0.012783999741077422
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,15,0.012639999389648438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,31,0.011828800290822982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,31,0.012835200130939483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,63,0.011766400188207626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,63,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,127,0.011876799911260606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,127,0.012891200184822083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,255,0.013779200613498688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,255,0.01467680037021637
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,511,0.016620799899101257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,511,0.018291200697422027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,1023,0.017795200645923614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,1023,0.017286400496959686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,2047,0.02003519982099533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,2047,0.020819200575351714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,4095,0.03648479878902435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,4095,0.02733759880065918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,8191,0.05835199952125549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,8191,0.042140799760818484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,1,0.01523360013961792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,float16,16383,0.100547194480896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,16,8,128,1,float16,fp8,16383,0.06714879870414733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,1,0.015427200496196747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,3,0.01462559998035431
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,3,0.01587519943714142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,7,0.01438400000333786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,15,0.014683200418949128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,7,0.015436799824237823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,15,0.015382400155067444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,31,0.01480640023946762
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,31,0.015304000675678253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,63,0.014664000272750855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,63,0.0154448002576828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,127,0.014777599275112152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,127,0.015451200306415558
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,255,0.016443200409412384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,255,0.017345599830150604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,511,0.020393599569797517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,511,0.02107200026512146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,1023,0.03851040005683899
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,1023,0.028718400001525878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,2047,0.06010079979896545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,2047,0.042664000391960145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,4095,0.10305279493331909
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,4095,0.06797119975090027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,1,0.018481600284576415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,1,0.019782400131225585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,float16,8191,0.18738240003585815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,3,0.018641600012779237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,3,0.01997919976711273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,1,128,1,float16,fp8,8191,0.1192080020904541
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,7,0.01813279986381531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,15,0.019883200526237488
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,31,0.018571199476718904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,15,0.018331199884414673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,7,0.019702400267124175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,31,0.019648000597953796
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,63,0.018411199748516082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,63,0.020049600303173064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,127,0.01820479929447174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,255,0.02228800058364868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,127,0.019750399887561797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,255,0.023900799453258514
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,511,0.031857600808143614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,511,0.039401599764823915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,1023,0.06311839818954468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,1023,0.046804800629615784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,2047,0.10586400032043457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,2047,0.07153279781341552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,4095,0.12207200527191162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,4095,0.1891711950302124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,1,0.025681599974632263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,3,0.02858240008354187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,1,0.02894560098648071
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,3,0.026259198784828186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,fp8,8191,0.22384800910949706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,2,128,1,float16,float16,8191,0.35622079372406007
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,7,0.025696000456809996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,7,0.02903040051460266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,15,0.025971201062202454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,31,0.02871519923210144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,15,0.029316800832748412
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,63,0.026137599349021913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,63,0.02906399965286255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,31,0.025775998830795288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,127,0.02720479965209961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,255,0.04476960003376007
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,255,0.03736160099506378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,127,0.02945440113544464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,511,0.06464319825172424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,511,0.054020798206329344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,1023,0.08098400235176087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,1023,0.10968639850616455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,2047,0.19139200448989868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,2047,0.12887359857559205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,1,0.0401632010936737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,4095,0.22530879974365234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,1,0.04702720046043396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,3,0.040214401483535764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,4095,0.35638720989227296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,7,0.04058560132980347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,7,0.04627839922904968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,3,0.0466592013835907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,15,0.04688000082969666
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,31,0.04008159935474396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,31,0.046744000911712644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,15,0.0406607985496521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,fp8,8191,0.42960638999938966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,4,128,1,float16,float16,8191,0.6891088008880615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,63,0.041223999857902524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,63,0.046300798654556274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,127,0.04867999851703644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,127,0.04813440144062042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,255,0.07298240065574646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,255,0.0677727997303009
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,511,0.11203680038452149
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,511,0.09433599710464477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,1023,0.19640640020370484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,1023,0.1454367995262146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,2047,0.24055840969085693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,2047,0.36405439376831056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,1,0.021320000290870667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,1,0.023137600719928743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,3,0.020983999967575072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,4095,0.4359903812408447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,3,0.023004800081253052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,7,0.021040000021457672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,4095,0.6933279991149902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,15,0.02086080014705658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,15,0.023175999522209167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,7,0.022353599965572356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,31,0.022563199698925018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,63,0.020883199572563172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,31,0.02147199958562851
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,63,0.022569599747657775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,127,0.020980800688266753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,127,0.022699199616909027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,255,0.026348799467086792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,255,0.025201600790023804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,511,0.04266240000724793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,511,0.035016000270843506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,fp8,8191,0.8284624099731446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,1023,0.06697760224342346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,1023,0.05050879716873169
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,float16,2047,0.10849599838256836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,1,128,1,float16,fp8,2047,0.07424319982528686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,1,0.030748799443244934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,3,0.028190401196479798
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,16,8,128,1,float16,float16,8191,1.3582768440246582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,1,0.02823199927806854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,3,0.03140319883823395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,7,0.028265601396560668
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,7,0.03062880039215088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,15,0.028281599283218384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,63,0.028327998518943787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,31,0.03126240074634552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,31,0.028431999683380126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,63,0.03065280020236969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,15,0.03308640122413635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,127,0.029361599683761598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,127,0.03120799958705902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,255,0.04764960110187531
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,511,0.05915039777755737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,511,0.06872959733009339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,1023,0.112881600856781
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,255,0.03978399932384491
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,1023,0.08335999846458435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,1,0.048767998814582825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,1,0.044331198930740355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,float16,2047,0.19426560401916504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,2,128,1,float16,fp8,2047,0.13129600286483764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,3,0.04338400065898895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,3,0.048209598660469054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,7,0.0431439995765686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,7,0.04878079891204834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,15,0.04414080083370209
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,31,0.04328320026397705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,15,0.049214398860931395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,63,0.049377599358558656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,31,0.049296000599861146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,63,0.04341920018196106
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,127,0.05112640261650085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,127,0.05024799704551697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,255,0.07596319913864136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,255,0.06836479902267456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,511,0.1167072057723999
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,511,0.09778079986572266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,1,0.07132959961891175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,1023,0.20190720558166503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,1023,0.14550880193710328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,1,0.08458399772644043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,3,0.07178720235824584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,3,0.08456000089645385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,7,0.07027999758720398
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,7,0.08344159722328186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,fp8,2047,0.24541919231414794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,15,0.07181280255317687
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,15,0.08498719930648804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,31,0.07189599871635437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,4,128,1,float16,float16,2047,0.42226881980895997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,31,0.083024001121521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,127,0.08743680119514466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,63,0.08775839805603028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,63,0.07808799743652343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,127,0.09555519819259643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,255,0.11780320405960083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,255,0.13572800159454346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,511,0.1796239972114563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,1,0.05082560181617737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,511,0.21317439079284667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,1,0.055164802074432376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,3,0.03427520096302032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,1023,0.27741758823394774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,3,0.03755840063095093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,1023,0.3689903974533081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,7,0.05086399912834168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,7,0.03726719915866852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,15,0.03753120005130768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,15,0.03448159992694855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,63,0.03443520069122315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,31,0.055561602115631104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,31,0.03454880118370056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,63,0.03742879927158356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,127,0.03587839901447296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,fp8,2047,0.4615824222564697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,127,0.037567999958992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,255,0.0468639999628067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,255,0.0650543987751007
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,16,8,128,1,float16,float16,2047,0.6986288070678711
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,511,0.06259999871253967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,511,0.07491199970245362
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,1,0.04922559857368469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,1,0.07559040188789368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,fp8,1023,0.08913599848747253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,1,128,1,float16,float16,1023,0.11984959840774537
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,3,0.05453280210494995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,7,0.05493599772453308
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,7,0.04896320104598999
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,3,0.04891040027141571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,15,0.06728799939155579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,15,0.05390080213546753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,31,0.04865759909152985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,63,0.05531200170516968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,63,0.050616002082824706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,31,0.05449600219726562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,127,0.057265597581863406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,127,0.06888800263404846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,255,0.081140798330307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,255,0.07209280133247375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,511,0.12253760099411011
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,511,0.10324000120162964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,1,0.09456800222396851
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,1,0.08921599984169007
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,3,0.09081119894981385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,3,0.07631840109825135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,float16,1023,0.20823678970336915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,2,128,1,float16,fp8,1023,0.15538719892501832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,7,0.07742080092430115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,7,0.11045759916305542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,15,0.07676640152931213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,15,0.09071679711341858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,31,0.0773248016834259
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,63,0.08111360073089599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,63,0.09353280067443848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,31,0.08978880047798157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,127,0.09811519980430602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,127,0.09795039892196655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,255,0.12574080228805543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,255,0.1243664026260376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,511,0.21314079761505128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,1,0.1307711958885193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,511,0.18371039628982544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,3,0.1314239978790283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,1,0.15985759496688842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,fp8,1023,0.2809999942779541
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,7,0.13090879917144777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,4,128,1,float16,float16,1023,0.3895312070846558
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,3,0.1552896022796631
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,15,0.13160959482192994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,7,0.15803359746932982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,15,0.1599984049797058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,31,0.157259202003479
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,31,0.13646399974822998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,127,0.15415040254592896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,63,0.1658079981803894
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,63,0.182750403881073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,127,0.17786879539489747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,255,0.22508320808410645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,255,0.22410399913787843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,1,0.01165440008044243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,1,0.012680000066757202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,3,0.012465599924325943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,7,0.011894399672746659
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,3,0.012008000165224075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,7,0.012488000094890594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,15,0.01183359995484352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,511,0.4095903873443604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,15,0.012700800597667695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,31,0.011975999921560287
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,511,0.3439232110977173
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,63,0.012039999663829803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,63,0.012708799540996551
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,31,0.012439999729394913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,127,0.012617599964141846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,255,0.013619199395179749
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,127,0.01173280030488968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,255,0.01438080072402954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,511,0.018348799645900728
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,511,0.016471999883651733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,1023,0.01608159989118576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,1023,0.016927999258041383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,2047,0.016577599942684172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,float16,1023,0.710200023651123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,2047,0.01714400053024292
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,4095,0.018454399704933167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,4095,0.018217599391937254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,16,8,128,1,float16,fp8,1023,0.779033613204956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,8191,0.02245599925518036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,float16,16383,0.03864319920539856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,1,0.011734399944543839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,16383,0.0283376008272171
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,1,128,1,float16,fp8,8191,0.022251200675964356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,3,0.011990399658679962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,7,0.011707200109958649
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,3,0.012292800098657608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,1,0.018356800079345703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,15,0.012611199915409089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,7,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,31,0.011753600090742111
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,31,0.012779200077056884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,15,0.011929599940776825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,63,0.01271039992570877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,63,0.01729599982500076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,127,0.011763200163841248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,255,0.013553600013256072
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,127,0.012443199753761292
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,255,0.01449120044708252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,511,0.01648160070180893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,511,0.02901119887828827
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,1023,0.016212800145149232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,1023,0.0168272003531456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,2047,0.017851200699806214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,2047,0.01764000058174133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,4095,0.02231999933719635
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,4095,0.0341264009475708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,8191,0.027668800950050355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,8191,0.037745600938796996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,1,0.011952000111341477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,1,0.019886399805545806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,float16,16383,0.059113597869873045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,3,0.011777599900960922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,3,0.012729600071907043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,7,0.011961600184440613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,2,128,1,float16,fp8,16383,0.041884800791740416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,7,0.01982239931821823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,15,0.011931200325489045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,15,0.012577599287033081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,31,0.012703999876976013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,31,0.011947199702262878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,63,0.017046399414539337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,63,0.01194240003824234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,127,0.012604799866676331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,127,0.011963199824094772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,255,0.013902400434017182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,255,0.014724799990653991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,511,0.02258719950914383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,511,0.018163199722766876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,1023,0.017459200322628023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,1023,0.0173007994890213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,2047,0.01998720020055771
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,2047,0.021062399446964263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,4095,0.040188801288604734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,4095,0.027137601375579835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,8191,0.058620798587799075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,1,0.015936000645160674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,1,0.012694400548934937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,8191,0.04235199987888336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,3,0.011788800358772278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,fp8,16383,0.06728320121765137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,3,0.016953599452972413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,4,128,1,float16,float16,16383,0.10140479803085327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,7,0.012164799869060517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,7,0.012934400141239167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,15,0.011771199852228164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,31,0.013780799508094788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,15,0.012806400656700134
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,63,0.012145599722862244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,63,0.012838399410247803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,31,0.012854400277137756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,127,0.01207360029220581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,127,0.014721600711345673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,255,0.013870400190353394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,255,0.01483680009841919
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,511,0.016953599452972413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,511,0.01847199946641922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,1023,0.01967200040817261
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,1023,0.024622400104999543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,2047,0.036766400933265685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,2047,0.027651199698448183
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,4095,0.05843520164489746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,4095,0.04200640022754669
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,8191,0.06714400053024291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,8191,0.10330079793930054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,1,0.05393120050430298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,fp8,16383,0.11676959991455078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,1,0.06090080142021179
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,16,8,128,1,float16,float16,16383,0.18448959589004515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,3,0.05917440056800842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,3,0.05982720255851746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,7,0.060996800661087036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,7,0.05460159778594971
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,15,0.054808002710342404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,15,0.06554239988327026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,31,0.05433279871940613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,31,0.060329598188400266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,63,0.058608001470565795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,63,0.0657263994216919
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,127,0.06301119923591614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,127,0.06343680024147033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,255,0.08417440056800843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,255,0.07831839919090271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,float16,511,0.13141920566558837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,1,0.08213919997215272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,1,128,1,float16,fp8,511,0.11043519973754883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,3,0.08287839889526367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,1,0.09534400105476379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,3,0.0942143976688385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,7,0.08940640091896057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,7,0.09396640062332154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,15,0.08280799984931946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,15,0.09405919909477234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,31,0.09446399807929992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,31,0.08512319922447205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,63,0.09424160122871399
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,63,0.09883840084075927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,127,0.09432160258293151
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,127,0.10227040052413941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,255,0.1344655990600586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,255,0.1295807957649231
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,1,0.13851360082626343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,float16,511,0.23660800457000733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,1,0.16146559715270997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,2,128,1,float16,fp8,511,0.1876863956451416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,3,0.13787519931793213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,3,0.16362719535827636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,7,0.1382464051246643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,15,0.13730720281600953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,31,0.14242559671401978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,7,0.20108799934387206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,15,0.16125439405441283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,63,0.14549119472503663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,31,0.16508159637451172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,63,0.17081120014190673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,127,0.15324000120162964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,127,0.23759360313415528
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,255,0.22019200325012206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,255,0.22867999076843262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,1,0.2416815996170044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,float16,511,0.40331521034240725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,4,128,1,float16,fp8,511,0.34370400905609133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,3,0.24102399349212647
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,1,0.4073887825012207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,3,0.2936431884765625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,7,0.2417680025100708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,7,0.2953775882720947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,15,0.29104158878326414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,15,0.25267040729522705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,31,0.347052788734436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,63,0.2592463970184326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,31,0.3158895969390869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,127,0.27456159591674806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,63,0.3173504114151001
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,127,0.3312432050704956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,1,0.0949679970741272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,1,0.10752639770507813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,3,0.09305279850959777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,255,0.4209136009216309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,255,0.5265727996826172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,3,0.10763360261917114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,7,0.09305599927902222
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,7,0.10592479705810547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,15,0.10775519609451294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,31,0.09547359943389892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,float16,511,0.8063743591308594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,15,0.16371999979019164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,63,0.1008031964302063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,31,0.10671360492706299
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,63,0.11395360231399536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,16,8,128,1,float16,fp8,511,0.6553487777709961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,127,0.10709279775619507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,float16,255,0.22632319927215577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,255,0.13886719942092896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,1,0.1480080008506775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,1,128,1,float16,fp8,127,0.118014395236969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,3,0.14700160026550294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,1,0.17348959445953369
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,3,0.17185920476913452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,15,0.14882240295410157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,7,0.1684720039367676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,15,0.17102880477905275
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,7,0.23998239040374755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,31,0.15569119453430175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,31,0.17514079809188843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,63,0.15905920267105103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,63,0.17968640327453614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,127,0.1684015989303589
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,float16,255,0.24087200164794922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,255,0.23850560188293457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,2,128,1,float16,fp8,127,0.29314560890197755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,1,0.26048479080200193
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,1,0.30139999389648436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,3,0.25860159397125243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,7,0.26019840240478515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,3,0.30995841026306153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,15,0.2645344018936157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,7,0.4606815814971924
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,15,0.3032047986984253
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,31,0.2659807920455933
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,31,0.32301759719848633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,63,0.2744784116744995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,63,0.32695679664611815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,127,0.32527039051055906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,127,0.3719759941101074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,float16,255,0.4177087783813477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,1,0.47059202194213867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,4,128,1,float16,fp8,255,0.43336000442504885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,3,0.47040162086486814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,1,0.5643743991851806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,7,0.48466877937316893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,15,0.48775520324707033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,3,0.822868824005127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,7,0.5706175804138184
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,15,0.5955296039581299
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,31,0.4879936218261719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,31,0.6091487884521485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,63,0.49703521728515626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,1,0.011851199716329575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,3,0.011763200163841248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,1,0.012542399764060973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,3,0.01266240030527115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,63,0.5983776092529297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,7,0.011851199716329575
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,7,0.012540799379348756
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,15,0.01250080019235611
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,127,0.6184095859527587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,15,0.01170239970088005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,31,0.011876799911260606
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,127,0.7201056003570556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,31,0.012507200241088867
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,63,0.01242239996790886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,127,0.01284160017967224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,63,0.011988800019025803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,255,0.01332319974899292
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,255,0.014262400567531586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,127,0.019131200015544893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,511,0.01676799952983856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,float16,255,0.7925087928771972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,511,0.018089599907398224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,1023,0.01632480025291443
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,1023,0.017049600183963776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,16,8,128,1,float16,fp8,255,0.8016063690185546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,2047,0.017990399897098542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,2047,0.029164800047874452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,4095,0.021190400421619415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,4095,0.021323199570178985
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,8191,0.037887999415397645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,8191,0.027089598774909972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,1,0.01808159947395325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,fp8,16383,0.042891201376914975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,1,128,1,float16,float16,16383,0.059487998485565186
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,3,0.01188800036907196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,1,0.012644800543785095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,7,0.019331200420856474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,3,0.01263359934091568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,7,0.01188800036907196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,31,0.011852800101041793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,31,0.012668800354003907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,15,0.012600000202655792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,15,0.012268800288438797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,63,0.01290079951286316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,63,0.018177600204944612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,127,0.01202080026268959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,127,0.012703999876976013
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,255,0.013702400028705597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,255,0.014499199390411378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,511,0.017015999555587767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,511,0.018355199694633485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,1023,0.026694399118423463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,1023,0.017108799517154695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,2047,0.020791999995708466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,2047,0.019865599274635316
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,4095,0.0372191995382309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,4095,0.028753599524497984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,8191,0.05870720148086548
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,8191,0.051051199436187744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,1,0.011915200203657151
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,1,0.012604799866676331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,float16,16383,0.10201760530471801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,3,0.012297599762678146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,2,128,1,float16,fp8,16383,0.06642879843711853
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,3,0.01276959925889969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,7,0.012750400602817536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,7,0.011913599818944931
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,15,0.0118928000330925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,15,0.012891200184822083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,31,0.012839999794960023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,31,0.01218079999089241
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,63,0.012027200311422348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,63,0.012665599584579468
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,127,0.012064000219106674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,127,0.012894399464130402
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,255,0.0135903999209404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,255,0.014907200634479523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,511,0.016763199865818024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,511,0.018222400546073915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,1023,0.01985439956188202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,1023,0.020615999400615693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,2047,0.026767998933792114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,2047,0.037191998958587644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,4095,0.058595198392868045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,4095,0.04089759886264801
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,8191,0.10138720273971558
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,1,0.012179200351238251
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,8191,0.0655023992061615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,1,0.012887999415397644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,3,0.012004800140857697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,7,0.012191999703645706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,3,0.012892800569534301
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,15,0.012303999811410903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,float16,16383,0.18583840131759644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,15,0.012734399735927581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,7,0.012700800597667695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,4,128,1,float16,fp8,16383,0.11581920385360718
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,31,0.012172800302505494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,31,0.01287200003862381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,63,0.012137600034475327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,63,0.012796799838542938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,127,0.012779200077056884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,127,0.012625600397586822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,255,0.014988799393177033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,255,0.014243200421333313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,511,0.01709440052509308
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,511,0.01828159987926483
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,1023,0.03540000021457672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,1023,0.02646239995956421
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,2047,0.05718719959259033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,2047,0.04094400107860565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,4095,0.09934080243110657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,4095,0.06566879749298096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,1,0.013979199528694152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,1,0.0133775994181633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,8191,0.11427359580993653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,8191,0.1837488055229187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,3,0.01311199963092804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,3,0.0139055997133255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,7,0.01332319974899292
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,15,0.013935999572277069
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,31,0.01297920048236847
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,7,0.01395840048789978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,15,0.012934400141239167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,31,0.014057600498199463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,fp8,16383,0.21144959926605225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,16,8,128,1,float16,float16,16383,0.3513328075408936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,63,0.013214400410652161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,63,0.01408800035715103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,127,0.013495999574661254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,255,0.014876799285411834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,127,0.014263999462127686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,511,0.01950560063123703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,255,0.016463999450206757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,1023,0.020712000131607056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,511,0.01803999990224838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,1023,0.02210559993982315
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,2047,0.03883680105209351
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,2047,0.028574401140213014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,4095,0.04235680103302002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,4095,0.05957440137863159
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,8191,0.06654239892959594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,8191,0.10273599624633789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,1,0.013235199451446533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,1,0.014112000167369843
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,16383,0.11754080057144164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,3,0.013289600610733032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,3,0.013944000005722046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,16383,0.18727680444717407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,7,0.01313920021057129
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,15,0.014046399295330048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,15,0.013548800349235534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,7,0.014241600036621093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,fp8,32767,0.21614561080932618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,31,0.01401599943637848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,63,0.013195200264453888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,31,0.013312000036239623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,63,0.014124800264835358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,1,128,1,float16,float16,32767,0.3588592052459717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,127,0.0131632000207901
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,255,0.015108799934387207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,127,0.014267200231552124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,255,0.015868799388408662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,511,0.0182096004486084
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,511,0.019551999866962433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,1023,0.037089601159095764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,1023,0.0270687997341156
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,2047,0.05889279842376709
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,2047,0.041947200894355774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,4095,0.06561599969863892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,4095,0.10239520072937011
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,8191,0.11740959882736206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,8191,0.18547840118408204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,1,0.017203199863433837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,1,0.018369600176811218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,16383,0.21749439239501953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,3,0.016944000124931337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,16383,0.35540480613708497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,3,0.01854880005121231
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,7,0.018355199694633485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,7,0.017272000014781953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,15,0.018585599958896637
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,31,0.016883200407028197
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,15,0.0173552006483078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,31,0.018883199989795686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,63,0.01722559928894043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,63,0.018355199694633485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,fp8,32767,0.4070112228393555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,127,0.017713600397109987
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,127,0.01905120015144348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,255,0.021059200167655945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,255,0.022303999960422517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,2,128,1,float16,float16,32767,0.6908095836639404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,511,0.0314191997051239
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,511,0.04113920032978058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,1023,0.06188160181045532
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,1023,0.04604319930076599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,2047,0.10330239534378052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,2047,0.07295680046081543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,4095,0.11918400526046753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,4095,0.18636159896850585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,8191,0.2173919916152954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,8191,0.35331358909606936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,1,0.011395200341939925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,1,0.01239679977297783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,3,0.011337599903345107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,3,0.012441600114107132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,16383,0.41632800102233886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,7,0.011383999884128571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,7,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,31,0.01162080019712448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,16383,0.686736011505127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,15,0.012166400253772736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,15,0.011591999977827071
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,31,0.01242400035262108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,63,0.012172800302505494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,127,0.012216000258922577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,127,0.011630400270223617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,255,0.012980799376964568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,63,0.011353600025177001
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,255,0.01422239989042282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,511,0.016128000617027283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,511,0.0179967999458313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,1023,0.016676799952983858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,1023,0.015267199277877808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,2047,0.0157600000500679
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,2047,0.016648000478744505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,4095,0.016686399281024934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,4095,0.017131200432777403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,8191,0.01807679980993271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,8191,0.018673600256443025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,fp8,32767,0.807588768005371
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,16383,0.020703999698162077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,16383,0.021294400095939636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,float16,32767,0.023286400735378264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,1,0.011713600158691407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,1,128,1,float16,fp8,32767,0.024527999758720397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,3,0.01125440001487732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,3,0.01252640038728714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,7,0.011686400324106217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,7,0.012120000272989272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,1,0.012204799801111221
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,15,0.011281599849462509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,15,0.012539200484752655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,8,4,128,1,float16,float16,32767,1.3581711769104003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,31,0.012411200255155564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,31,0.011718399822711945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,63,0.011451199650764465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,127,0.011377599835395814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,127,0.012532800436019897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,255,0.013249599933624267
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,63,0.012188799679279327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,511,0.016251200437545778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,511,0.01799200028181076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,1023,0.016763199865818024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,2047,0.017132799327373504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,1023,0.015273599326610566
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,4095,0.016967999935150146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,2047,0.015855999290943147
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,255,0.01414719969034195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,8191,0.01786559969186783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,4095,0.017929600179195405
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,8191,0.018755200505256652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,16383,0.021063999831676485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,16383,0.02052319943904877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,1,0.011727999895811081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,fp8,32767,0.024268800020217897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,3,0.011716800183057785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,1,0.012375999987125397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,3,0.012144000083208085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,7,0.011687999963760376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,7,0.011999999731779098
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,15,0.012307199835777282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,15,0.011667200177907944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,31,0.011446399986743927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,2,128,1,float16,float16,32767,0.02449280023574829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,31,0.012432000041007996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,63,0.011473599821329117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,63,0.012292800098657608
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,127,0.011763200163841248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,127,0.012161599844694138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,255,0.01326719969511032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,511,0.01606079936027527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,255,0.014252799749374389
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,511,0.018185600638389587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,1023,0.016646400094032288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,1023,0.015831999480724335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,2047,0.01615840047597885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,4095,0.016918399930000307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,2047,0.017049600183963776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,4095,0.01764480024576187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,8191,0.0191103994846344
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,8191,0.018671999871730804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,16383,0.02311040014028549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,16383,0.02317280024290085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,1,0.01173119992017746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,fp8,32767,0.03118560016155243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,1,0.01210239976644516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,8,4,128,1,float16,float16,32767,0.038624000549316403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,3,0.011574400216341018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,3,0.01236959993839264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,7,0.01167680025100708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,15,0.011363200098276138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,7,0.012345600128173827
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,15,0.012060800194740295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,31,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,31,0.011615999788045884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,63,0.01173119992017746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,63,0.01233920007944107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,127,0.01149120032787323
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,127,0.012163200229406358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,255,0.013329599797725678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,255,0.014276799559593201
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,511,0.01629440039396286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,511,0.018171200156211854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,1023,0.01542080044746399
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,1023,0.016462400555610657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,2047,0.01637600064277649
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,2047,0.01741439998149872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,4095,0.01709440052509308
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,4095,0.017759999632835387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,8191,0.01799360066652298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,8191,0.018739199638366698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,16383,0.02094399929046631
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,16383,0.02117439955472946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,float16,32767,0.024932800233364104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,1,0.011321599781513213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,1,0.012008000165224075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,3,0.011598400026559829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,1,128,1,float16,fp8,32767,0.024641600251197816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,3,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,7,0.012177599966526032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,7,0.011769600212574005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,15,0.01138719990849495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,15,0.012200000137090683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,31,0.011488000303506852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,31,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,63,0.011409600079059602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,63,0.0124719999730587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,127,0.011400000005960465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,127,0.012334399670362473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,255,0.013153600692749023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,255,0.014396800100803376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,511,0.015992000699043274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,511,0.018187199532985688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,2047,0.01597920060157776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,1023,0.017003199458122252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,1023,0.015740799903869628
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,2047,0.017207999527454377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,4095,0.01685599982738495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,4095,0.017796799540519714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,8191,0.019038400053977965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,8191,0.019073599576950075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,16383,0.022908799350261688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,16383,0.022801600396633148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,1,0.011627200245857238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,fp8,32767,0.028987199068069458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,2,128,1,float16,float16,32767,0.0387935996055603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,1,0.012345600128173827
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,3,0.011355199664831162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,7,0.011561600118875503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,3,0.012321600317955017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,15,0.012108799815177918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,7,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,15,0.011523199826478958
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,31,0.011774399876594543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,31,0.01218400001525879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,127,0.011532799899578094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,63,0.0123648002743721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,63,0.011435200273990632
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,127,0.012328000366687774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,255,0.013171200454235078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,255,0.014083200693130493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,1023,0.016075199842453
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,511,0.016303999722003935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,511,0.017671999335289002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,1023,0.01693120002746582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,2047,0.016195200383663177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,2047,0.017297600209712983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,4095,0.01834239959716797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,4095,0.017897599935531618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,8191,0.02173919975757599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,8191,0.0216048002243042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,16383,0.037745600938796996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,16383,0.028097599744796753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,float16,32767,0.05975199937820434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,1,0.012415999919176102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,1,0.013259199261665345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,3,0.013471999764442444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,3,0.012721599638462066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,8,4,128,1,float16,fp8,32767,0.0443583995103836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,7,0.013052800297737121
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,7,0.012604799866676331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,15,0.01250240057706833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,15,0.013335999846458436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,31,0.012803199887275695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,31,0.013355199992656708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,63,0.01244639977812767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,63,0.0133775994181633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,127,0.013419200479984284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,127,0.012771199643611907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,255,0.014475199580192565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,255,0.015617600083351136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,511,0.017052799463272095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,511,0.018884800374507904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,1023,0.0179967999458313
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,1023,0.018387199938297273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,2047,0.02080159932374954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,2047,0.02191839963197708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,4095,0.03790239989757538
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,4095,0.02871040105819702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,8191,0.05929440259933472
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,8191,0.0435232013463974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,16383,0.10266720056533814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,16383,0.06781280040740967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,1,0.01265760064125061
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,7,0.012620800733566284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,1,0.013439999520778656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,float16,32767,0.18731679916381835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,1,128,1,float16,fp8,32767,0.11845920085906983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,3,0.01266079992055893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,3,0.013660800457000733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,7,0.013315199315547943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,15,0.012804800271987915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,15,0.013156799972057343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,31,0.01358720064163208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,63,0.012787200510501862
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,127,0.012915199995040894
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,63,0.013449600338935852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,31,0.01284639984369278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,255,0.014660799503326416
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,127,0.01324159950017929
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,511,0.017692799866199493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,255,0.01563519984483719
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,511,0.018934400379657747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,1023,0.020552000403404234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,1023,0.021622399985790252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,2047,0.0278656005859375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,2047,0.03762399852275848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,4095,0.05941759943962097
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,4095,0.042438399791717527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,8191,0.10198400020599366
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,8191,0.06759359836578369
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,16383,0.11792960166931152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,16383,0.18681759834289552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,1,0.01279360055923462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,1,0.013808000087738036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,3,0.012521600723266602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,7,0.01265919953584671
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,7,0.013780799508094788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,fp8,32767,0.21644799709320067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,3,0.013731199502944946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,15,0.01249919980764389
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,15,0.01363999992609024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,31,0.012939199805259705
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,2,128,1,float16,float16,32767,0.35612480640411376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,63,0.01767839938402176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,63,0.013232000172138214
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,127,0.013769599795341491
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,31,0.013617600500583648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,127,0.012830400466918945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,255,0.021740800142288207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,511,0.017499199509620665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,255,0.014628799259662628
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,511,0.019388799369335175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,1023,0.03634240031242371
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,1023,0.03806079924106598
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,2047,0.04158560037612915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,2047,0.057871997356414795
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,4095,0.10082399845123291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,4095,0.06704800128936768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,8191,0.18972159624099733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,8191,0.11650079488754272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,1,0.01170400008559227
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,16383,0.21461279392242433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,1,0.012374400347471236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,16383,0.3527456045150757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,3,0.012353599816560746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,3,0.01154239997267723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,7,0.01231359988451004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,7,0.011404799669981003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,15,0.011644800007343293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,15,0.012347199767827988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,31,0.011478400230407715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,31,0.012403199821710587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,63,0.01148959994316101
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,float16,32767,0.6893104076385498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,127,0.011580800265073776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,63,0.012187200039625168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,127,0.012329600006341934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,255,0.014287999272346497
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,511,0.0160288006067276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,8,4,128,1,float16,fp8,32767,0.6318111896514893
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,511,0.017980800569057466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,255,0.013332800567150116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,1023,0.015964800119400026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,1023,0.016657599806785585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,2047,0.01717599928379059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,2047,0.025281599164009093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,4095,0.017854399979114532
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,8191,0.0189983993768692
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,4095,0.016940799355506898
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,8191,0.0192208006978035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,16383,0.022841599583625794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,16383,0.04235840141773224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,1,0.011494400352239609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,float16,32767,0.03915840089321136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,1,128,1,float16,fp8,32767,0.029769599437713623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,1,0.012361600250005721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,3,0.012347199767827988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,3,0.021745599806308746
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,7,0.011360000073909759
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,15,0.011787199974060058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,15,0.012332800030708312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,7,0.012211199849843979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,31,0.021668800711631776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,31,0.012124799937009812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,63,0.011585599929094314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,63,0.012361600250005721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,127,0.011683200299739838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,127,0.023291200399398804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,255,0.013031999766826629
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,255,0.014361600577831268
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,511,0.01615999937057495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,511,0.01791999936103821
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,1023,0.029956799745559693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,1023,0.016711999475955964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,2047,0.01637279987335205
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,2047,0.017318400740623473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,4095,0.029815998673439027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,4095,0.01839520037174225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,8191,0.02141920030117035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,8191,0.02163680046796799
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,16383,0.038043200969696045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,16383,0.04428640007972717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,1,0.012167999893426895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,1,0.011675199866294861
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,float16,32767,0.059824001789093015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,3,0.011761599779129028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,2,128,1,float16,fp8,32767,0.04524160027503967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,3,0.019601599872112276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,7,0.011539199948310852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,7,0.01231200024485588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,15,0.011558400094509124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,15,0.0127920001745224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,31,0.01944800019264221
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,31,0.011691199988126755
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,63,0.011503999680280685
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,63,0.012289600074291229
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,127,0.012484800070524216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,127,0.011604800075292587
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,255,0.014548799395561219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,255,0.020839999616146087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,511,0.016171200573444365
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,511,0.01794240027666092
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,1023,0.016326400637626647
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,1023,0.01698880046606064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,2047,0.02205439954996109
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,2047,0.01780800074338913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,4095,0.021158400177955627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,4095,0.021328000724315642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,8191,0.0370959997177124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,8191,0.027063998579978942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,16383,0.0618511974811554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,16383,0.041576001048088077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,1,0.014396800100803376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,1,0.019312000274658202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,fp8,32767,0.06744959950447083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,8,4,128,1,float16,float16,32767,0.1012384057044983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,3,0.015092800557613372
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,3,0.015227200090885162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,7,0.01836480051279068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,15,0.01520799994468689
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,7,0.01584160029888153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,31,0.014454400539398194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,15,0.015731200575828552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,31,0.019387200474739075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,63,0.015035200119018554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,63,0.01594880074262619
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,127,0.014763200283050537
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,127,0.015745599567890168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,255,0.019300800561904908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,255,0.017580799758434296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,511,0.020601600408554077
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,511,0.020950399339199066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,1023,0.03835839927196503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,1023,0.03427520096302032
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,2047,0.06063519716262818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,2047,0.043347200751304625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,4095,0.10281120538711548
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,4095,0.06802080273628235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,8191,0.11798559427261353
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,1,0.01866399943828583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,8191,0.18995679616928102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,3,0.018598400056362152
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,1,0.020190399885177613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,3,0.023294399678707122
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,float16,16383,0.3559135913848877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,1,128,1,float16,fp8,16383,0.21716480255126952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,7,0.018475200235843658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,7,0.019806399941444397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,15,0.021296000480651854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,31,0.018486399948596955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,63,0.020076799392700195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,127,0.02141280025243759
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,63,0.018529599905014037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,15,0.019844800233840942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,127,0.020153599977493285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,31,0.019649599492549897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,255,0.02236640006303787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,255,0.028046399354934692
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,511,0.039771199226379395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,511,0.03179199993610382
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,1023,0.047207999229431155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,1023,0.06280159950256348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,2047,0.10516480207443238
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,2047,0.08486559987068176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,4095,0.12318240404129029
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,4095,0.18794239759445192
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,1,0.02651520073413849
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,8191,0.22099359035491944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,1,0.028521600365638732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,8191,0.3548959970474243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,3,0.02637760043144226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,7,0.025740799307823182
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,7,0.02949439883232117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,3,0.029172798991203307
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,15,0.028491199016571045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,15,0.025617599487304688
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,31,0.026607999205589296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,31,0.029411199688911437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,63,0.025913599133491515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,63,0.028574401140213014
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,127,0.02746880054473877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,127,0.029241600632667543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,fp8,16383,0.5897840023040771
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,2,128,1,float16,float16,16383,0.689305591583252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,255,0.036792001128196715
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,255,0.04373280107975006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,511,0.06465920209884643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,511,0.054390400648117065
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,1023,0.10840480327606201
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,1023,0.13866080045700074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,2047,0.12707040309906006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,2047,0.1894703984260559
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,4095,0.22677600383758545
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,4095,0.3553839921951294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,1,0.02067199945449829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,1,0.022313599288463593
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,3,0.022300800681114195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,3,0.021092799305915833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,7,0.020926399528980254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,8191,0.4232816219329834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,7,0.022168000042438508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,8191,0.6890448093414306
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,15,0.022358399629592896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,15,0.021196800470352172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,31,0.022353599965572356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,31,0.020710399746894835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,63,0.02234559953212738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,127,0.020846399664878845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,127,0.022177599370479584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,63,0.021427200734615327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,255,0.024636800587177276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,255,0.02624320089817047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,511,0.043105599284172055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,511,0.03497759997844696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,1023,0.06627039909362793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,1023,0.050337600708007815
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,2047,0.10781760215759277
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,2047,0.07404639720916747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,fp8,16383,0.8132096290588379
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,1,0.027937600016593934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,float16,4095,0.19117920398712157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,1,0.03118720054626465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,1,128,1,float16,fp8,4095,0.1234671950340271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,3,0.03070240020751953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,3,0.027982398867607117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,7,0.031139200925827025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,7,0.028121599555015565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,15,0.02816160023212433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,15,0.030851200222969055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,31,0.031225600838661195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,63,0.028279998898506166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,63,0.030931198596954347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,127,0.0283376008272171
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,31,0.0280815988779068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,127,0.031324800848960874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,255,0.04643999934196472
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,255,0.03939360082149505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,511,0.06714079976081848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,511,0.0555728018283844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,1023,0.0826799988746643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,1023,0.11152960062026977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,8,4,128,1,float16,float16,16383,1.978144073486328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,2047,0.19296640157699585
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,2047,0.12893600463867189
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,1,0.042475199699401854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,7,0.04244639873504639
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,float16,4095,0.3602288007736206
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,3,0.04344640076160431
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,3,0.04938719868659973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,7,0.048214399814605714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,1,0.191046404838562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,2,128,1,float16,fp8,4095,0.22715039253234864
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,15,0.04927360117435455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,31,0.04335359930992126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,15,0.04332320094108581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,31,0.04838399887084961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,63,0.04837439954280853
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,127,0.05158720016479492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,127,0.04953120052814484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,255,0.06738399863243102
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,255,0.07523199915885925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,511,0.11563999652862549
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,63,0.16613600254058838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,511,0.09582080245018006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,1,0.0338128000497818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,1023,0.2004080057144165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,1023,0.14643679857254027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,1,0.0369376003742218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,3,0.0342848002910614
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,2047,0.24361600875854492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,3,0.03713760077953339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,7,0.03375039994716644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,2047,0.3597280025482178
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,7,0.037376001477241516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,15,0.0367935985326767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,31,0.03409439921379089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,15,0.03468959927558899
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,63,0.03445119857788086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,63,0.037064000964164734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,31,0.03749600052833557
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,127,0.037750399112701415
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,127,0.03472479879856109
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,float16,4095,0.690776014328003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,255,0.051988798379898074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,255,0.04550400078296661
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,511,0.0624239981174469
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,511,0.07373759746551514
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,1023,0.11882239580154419
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,1023,0.08946080207824707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,1,0.04869279861450195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,fp8,2047,0.13538880348205568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,1,0.05459679961204529
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,8,4,128,1,float16,fp8,4095,0.7097296237945556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,1,128,1,float16,float16,2047,0.20095200538635255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,3,0.048363199830055235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,7,0.048702400922775266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,3,0.053646397590637204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,15,0.04898560047149658
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,7,0.05372639894485474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,31,0.0539792001247406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,31,0.04892959892749786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,63,0.049384000897407535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,63,0.05363199710845947
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,15,0.0872879981994629
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,127,0.05657920241355896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,127,0.055508798360824584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,255,0.0804639995098114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,255,0.07289119958877563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,511,0.10132800340652466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,511,0.12202240228652954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,1023,0.15481280088424682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,1023,0.23146560192108154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,1,0.0763264000415802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,1,0.08958719968795777
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,7,0.0773743987083435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,3,0.07747359871864319
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,7,0.0896224021911621
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,fp8,2047,0.2489487886428833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,3,0.1343407988548279
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,15,0.07737759947776794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,2,128,1,float16,float16,2047,0.36700479984283446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,15,0.08996319770812988
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,31,0.07700639963150024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,31,0.0900592029094696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,63,0.09250239729881286
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,63,0.11487040519714356
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,127,0.09700959920883179
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,127,0.08501120209693909
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,255,0.12583359479904174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,255,0.12423360347747803
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,511,0.21214399337768555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,511,0.26559040546417234
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,1,0.011956799775362015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,1023,0.28054080009460447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,1,0.016577599942684172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,1023,0.3718960046768188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,3,0.011847999691963196
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,7,0.01570879966020584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,15,0.011828800290822982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,3,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,15,0.01676799952983856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,7,0.012492799758911132
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,31,0.011897599697113037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,31,0.01241919994354248
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,63,0.015729600191116334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,63,0.01263359934091568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,127,0.011899200081825257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,127,0.012495999783277511
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,255,0.017972800135612487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,fp8,2047,0.47302398681640623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,255,0.014584000408649444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,8,4,128,1,float16,float16,2047,0.701313591003418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,511,0.016364799439907075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,511,0.01802240014076233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,1023,0.02149759978055954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,2047,0.01666080057621002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,4095,0.018382400274276733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,1023,0.01727679967880249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,4095,0.01808159947395325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,8191,0.025332799553871153
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,8191,0.02253119945526123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,2047,0.01746080070734024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,16383,0.02848159968852997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,16383,0.03818399906158447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,1,0.012787200510501862
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,float16,32767,0.060476797819137576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,1,128,1,float16,fp8,32767,0.05227040052413941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,3,0.011844799667596818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,7,0.012094400078058242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,3,0.0125231996178627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,7,0.012433599680662155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,31,0.011910399794578553
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,15,0.011720000207424164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,1,0.011953599750995636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,63,0.011793600022792816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,15,0.013920000195503235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,31,0.012721599638462066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,63,0.012380799651145935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,127,0.011798399686813354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,127,0.0123648002743721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,255,0.01568640023469925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,511,0.01661760061979294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,255,0.013643200695514678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,1023,0.01626719981431961
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,511,0.018192000687122345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,1023,0.017190399765968322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,2047,0.01759839951992035
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,2047,0.018956799805164338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,4095,0.021638399362564086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,4095,0.02152799963951111
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,8191,0.03761279881000519
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,8191,0.02800160050392151
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,16383,0.04272960126399994
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,16383,0.05877280235290527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,1,0.012092799693346024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,float16,32767,0.10329600572586059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,3,0.012014400213956833
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,1,0.0130048006772995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,2,128,1,float16,fp8,32767,0.06782400012016296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,3,0.012566399574279786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,7,0.011902400106191636
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,7,0.013009600341320038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,15,0.012785600125789642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,15,0.012494400143623352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,31,0.012329600006341934
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,63,0.011934400349855424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,63,0.013121600449085235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,127,0.012104000151157378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,31,0.012761600315570831
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,127,0.012700800597667695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,255,0.014273600280284881
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,255,0.015046399831771851
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,511,0.0171984001994133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,511,0.018622399866580965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,1023,0.01772159934043884
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,1023,0.017324799299240114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,2047,0.02078240066766739
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,2047,0.02151840031147003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,4095,0.027531200647354127
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,4095,0.03684479892253876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,8191,0.058481597900390626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,8191,0.04181439876556396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,16383,0.06705600023269653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,16383,0.10155999660491943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,1,0.05517280101776123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,1,0.06028640270233154
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,3,0.0560591995716095
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,float16,32767,0.18559999465942384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,3,0.06079999804496765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,7,0.055164802074432376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,8,4,128,1,float16,fp8,32767,0.11862080097198487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,7,0.06028320193290711
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,15,0.056004798412323
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,31,0.055088001489639285
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,15,0.06135680079460144
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,31,0.06084960103034973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,63,0.05752159953117371
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,63,0.0606719970703125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,127,0.06308799982070923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,127,0.06306880116462707
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,255,0.08575519919395447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,255,0.07844319939613342
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,511,0.12439680099487305
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,511,0.10964000225067139
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,1,0.0836896002292633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,float16,1023,0.21609439849853515
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,1,0.0950160026550293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,3,0.08361920118331909
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,1,128,1,float16,fp8,1023,0.1626271963119507
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,3,0.09400960206985473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,7,0.08364959955215454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,15,0.08364480137825012
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,7,0.10235840082168579
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,15,0.09538559913635254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,31,0.08375840187072754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,31,0.09439520239830017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,63,0.08839359879493713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,63,0.09859520196914673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,127,0.11302560567855835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,127,0.1028704047203064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,255,0.13517760038375853
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,255,0.12797759771347045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,511,0.21488640308380128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,511,0.19122560024261476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,1,0.13932960033416747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,1,0.16165119409561157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,3,0.16352959871292114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,float16,1023,0.40743842124938967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,3,0.1393936038017273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,2,128,1,float16,fp8,1023,0.2916255950927734
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,7,0.1374079942703247
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,15,0.13763519525527954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,7,0.16335519552230834
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,31,0.1429744005203247
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,63,0.14424799680709838
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,31,0.16223200559616088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,127,0.15013279914855956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,63,0.1732624053955078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,15,0.2369152069091797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,127,0.1765936017036438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,255,0.21983039379119873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,1,0.0948639988899231
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,255,0.22651679515838624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,1,0.10695359706878663
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,3,0.09486079812049866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,511,0.3978127956390381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,511,0.34658079147338866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,7,0.09490560293197632
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,3,0.10558240413665772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,7,0.10683679580688477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,15,0.09379199743270875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,15,0.10525120496749878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,31,0.10562560558319092
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,31,0.09512959718704224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,63,0.10197759866714477
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,float16,1023,0.7121232032775879
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,63,0.11045759916305542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,127,0.10714399814605713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,127,0.11319839954376221
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,8,4,128,1,float16,fp8,1023,0.7923840045928955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,255,0.1434607982635498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,255,0.1420207977294922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,1,0.15028799772262574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,float16,511,0.23282079696655272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,1,128,1,float16,fp8,511,0.20163679122924805
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,1,0.1700144052505493
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,7,0.15052319765090943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,3,0.17050559520721437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,3,0.22418880462646484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,7,0.17298239469528198
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,15,0.14948639869689942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,15,0.17084319591522218
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,31,0.15350719690322875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,31,0.17134879827499389
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,63,0.15639840364456176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,127,0.16630239486694337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,127,0.18672800064086914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,63,0.27550880908966063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,255,0.2373296022415161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,255,0.23946080207824708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,float16,511,0.40063838958740233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,1,0.2622607946395874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,1,0.30890560150146484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,2,128,1,float16,fp8,511,0.3559808015823364
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,7,0.2621471881866455
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,3,0.30392639636993407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,3,0.3643984079360962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,15,0.2633280038833618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,7,0.3042736053466797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,31,0.26464641094207764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,15,0.3096560001373291
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,63,0.26818079948425294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,31,0.3228176116943359
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,127,0.2807919979095459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,63,0.4318992137908936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,1,0.0144896000623703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,127,0.3279520034790039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,1,0.012534399330615998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,3,0.014742399752140044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,7,0.011934400349855424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,3,0.012491200119256973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,7,0.01525759994983673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,15,0.01191840022802353
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,255,0.41196479797363283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,31,0.011772800236940384
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,15,0.015041600167751312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,255,0.43576641082763673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,31,0.012756800651550293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,127,0.014265599846839904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,63,0.012548799812793731
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,63,0.011860799789428712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,255,0.01361120045185089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,255,0.017668800055980684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,511,0.016497600078582763
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,127,0.012881599366664886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,511,0.018192000687122345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,1023,0.016415999829769136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,fp8,511,0.6774799823760986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,2047,0.01775840073823929
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,8,4,128,1,float16,float16,511,0.7791935920715332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,1023,0.02089280039072037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,2047,0.01804320067167282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,4095,0.02215680032968521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,4095,0.021529600024223328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,8191,0.03824959993362427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,8191,0.02831200063228607
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,1,0.012070400267839431
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,16383,0.06025599837303162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,float16,32767,0.10288480520248414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,1,0.012518399953842163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,32767,0.06838880181312561
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,1,128,1,float16,fp8,16383,0.042126399278640744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,3,0.011964800208806992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,3,0.012945599853992462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,15,0.011776000261306763
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,7,0.012964800000190735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,7,0.012966400384902954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,63,0.012174399942159653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,31,0.012556800246238708
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,63,0.013681599497795105
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,15,0.012724800407886505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,31,0.01196800023317337
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,127,0.011771199852228164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,127,0.012937599420547485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,255,0.01398719996213913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,511,0.01834239959716797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,255,0.014393599331378936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,511,0.01826079934835434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,1023,0.018038399517536163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,1023,0.01736160069704056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,2047,0.020268799364566804
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,2047,0.021201600134372712
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,4095,0.03766080141067505
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,4095,0.02672800123691559
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,8191,0.05889120101928711
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,8191,0.04232159852981567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,16383,0.10224000215530396
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,1,0.011811199784278869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,1,0.012732799351215362
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,16383,0.06637120246887207
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,3,0.012246400117874146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,fp8,32767,0.12263519763946533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,2,128,1,float16,float16,32767,0.1860319972038269
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,7,0.011777599900960922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,3,0.012768000364303589
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,7,0.012740799784660339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,15,0.013219200074672699
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,15,0.012035199999809265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,31,0.012627199292182922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,63,0.012070400267839431
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,31,0.012827199697494508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,127,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,63,0.013120000064373017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,255,0.013662399351596832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,127,0.012878400087356568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,255,0.014801600575447082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,511,0.018105599284172057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,511,0.018673600256443025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,1023,0.01974720060825348
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,1023,0.020468799769878386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,2047,0.037355199456214905
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,2047,0.0273360013961792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,4095,0.05958399772644043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,4095,0.04209280014038086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,8191,0.10174239873886108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,8191,0.06605920195579529
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,16383,0.11784800291061401
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,1,0.013689599931240082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,16383,0.18524479866027832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,1,0.013748799264431
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,3,0.014327999949455262
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,3,0.013531200587749481
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,7,0.01372320055961609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,7,0.01287200003862381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,fp8,32767,0.2163167953491211
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,15,0.013670399785041809
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,8,4,128,1,float16,float16,32767,0.3562623977661133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,15,0.014192000031471252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,31,0.013400000333786011
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,31,0.0139055997133255
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,127,0.013439999520778656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,255,0.014923200011253357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,127,0.014281600713729858
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,255,0.015897600352764128
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,511,0.018432000279426576
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,63,0.01496479958295822
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,63,0.014151999354362487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,511,0.019487999379634857
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,1023,0.02343519926071167
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,1023,0.022987200319766997
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,2047,0.044305598735809325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,2047,0.03951520025730133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,4095,0.06084640026092529
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,4095,0.04382080137729645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,8191,0.10538239479064941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,8191,0.06959840059280395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,16383,0.12540160417556762
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,16383,0.2031167984008789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,1,0.013494400680065155
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,32767,0.22241439819335937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,1,0.014486399292945863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,32767,0.36216959953308103
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,3,0.014204800128936768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,3,0.018670399487018586
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,7,0.014483200013637542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,7,0.013625599443912506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,15,0.020046399533748628
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,31,0.013475200533866883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,31,0.01433439999818802
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,63,0.01361120045185089
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,15,0.0134320005774498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,127,0.013740800321102142
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,63,0.020027199387550355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,127,0.014537599682807923
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,fp8,65535,0.4222080230712891
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,255,0.015780800580978395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,511,0.019200000166893005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,1,128,1,float16,float16,65535,0.6991407871246338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,255,0.016433599591255187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,511,0.028222399950027465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,1023,0.03751200139522552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,1023,0.027910399436950683
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,2047,0.058289599418640134
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,2047,0.04111199975013733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,8191,0.19018720388412474
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,4095,0.10225759744644165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,4095,0.06690880060195922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,8191,0.117958402633667
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,1,0.011537600308656693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,16383,0.35397920608520506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,16383,0.21781439781188966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,1,0.012328000366687774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,7,0.01154239997267723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,3,0.011665599793195725
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,3,0.012328000366687774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,7,0.01223360002040863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,15,0.011598400026559829
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,15,0.012417600303888322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,31,0.01162559986114502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,31,0.012412799894809723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,63,0.011460799723863602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,63,0.012305600196123123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,127,0.011395200341939925
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,32767,0.5527455806732178
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,32767,0.6904047966003418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,127,0.01228479966521263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,255,0.013203200697898865
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,511,0.01606079936027527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,1023,0.016702400147914888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,255,0.023052799701690673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,1023,0.017641599476337432
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,511,0.017972800135612487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,2047,0.016791999340057373
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,4095,0.0169295996427536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,2047,0.028974398970603943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,8191,0.018827199935913086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,8191,0.019491200149059296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,16383,0.036241599917411806
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,4095,0.01783040016889572
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,fp8,65535,0.8072128295898438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,16383,0.02322240024805069
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,32767,0.023848000168800353
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,32767,0.039587199687957764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,float16,65535,0.027635198831558228
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,1,0.011687999963760376
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,3,0.012140800058841706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,3,0.016884799301624297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,1,0.012176000326871873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,7,0.011404799669981003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,7,0.012478400021791458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,1,128,1,float16,fp8,65535,0.028331199288368226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,4,2,128,1,float16,float16,65535,1.3641936302185058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,15,0.011425600200891495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,15,0.01818079948425293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,31,0.011532799899578094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,31,0.012172800302505494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,63,0.011608000099658965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,63,0.012124799937009812
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,255,0.014185599982738495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,255,0.019364799559116363
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,511,0.015992000699043274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,127,0.012583999335765839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,1023,0.016628800332546233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,511,0.018529599905014037
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,2047,0.017688000202178956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,1023,0.017633600533008574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,127,0.011740799993276596
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,4095,0.01720000058412552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,2047,0.02463199943304062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,8191,0.018857599794864656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,4095,0.018160000443458557
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,8191,0.01958879977464676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,16383,0.02048960030078888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,32767,0.029123198986053467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,16383,0.021118399500846863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,32767,0.024728000164031982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,1,0.011372800171375274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,1,0.012273599952459335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,float16,65535,0.04086079895496368
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,4,2,128,1,float16,fp8,65535,0.032671999931335446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,3,0.01329759955406189
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,7,0.0121568001806736
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,15,0.011513599753379821
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,3,0.012336000055074691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,7,0.011668799817562104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,15,0.012164799869060517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,31,0.014351999759674073
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,31,0.011723200231790543
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,63,0.011524800211191177
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,63,0.012220799922943115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,127,0.011577600240707397
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,127,0.012428800016641617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,255,0.013233600556850434
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,255,0.01671359986066818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,511,0.016047999262809753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,511,0.018143999576568603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,1023,0.01666080057621002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,1023,0.01804639995098114
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,2047,0.017931200563907623
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,2047,0.016889600455760954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,4095,0.01995840072631836
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,4095,0.017980800569057466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,8191,0.01884640008211136
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,8191,0.01972319930791855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,16383,0.023684799671173096
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,16383,0.02311680018901825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,32767,0.02956640124320984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,1,0.0115167997777462
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,float16,65535,0.04283519983291626
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,32767,0.02746720016002655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,1,0.01234399974346161
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,3,0.011495999991893768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,1,128,1,float16,fp8,65535,0.032888001203536986
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,7,0.011606399714946748
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,3,0.013532799482345582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,7,0.012246400117874146
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,15,0.011838400363922119
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,15,0.01239520013332367
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,31,0.011670400202274323
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,63,0.011580800265073776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,63,0.013515199720859527
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,31,0.01236959993839264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,127,0.011521600186824799
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,127,0.012595200538635254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,255,0.013212800025939941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,255,0.014371199905872345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,511,0.019892799854278564
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,511,0.016201600432395935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,1023,0.016734400391578676
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,1023,0.0178384006023407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,2047,0.016952000558376312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,2047,0.018276800215244294
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,4095,0.018702399730682374
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,4095,0.018119999766349794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,8191,0.020110400021076204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,8191,0.0204352006316185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,16383,0.024208000302314757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,32767,0.04022560119628906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,16383,0.02369920015335083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,32767,0.030380800366401672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,1,0.012699200212955475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,float16,65535,0.061371201276779176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,3,0.012763200700283051
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,1,0.013831999897956849
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,4,2,128,1,float16,fp8,65535,0.04659680128097534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,7,0.013471999764442444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,7,0.012254399806261062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,3,0.01326880007982254
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,15,0.01276639997959137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,15,0.013939200341701508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,31,0.01244800016283989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,31,0.013544000685214996
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,63,0.0124208003282547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,63,0.013283200562000275
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,127,0.014070400595664978
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,127,0.012755200266838074
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,255,0.014350399374961853
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,255,0.01536799967288971
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,511,0.01712159961462021
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,511,0.01911199986934662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,1023,0.02027679979801178
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,1023,0.019420799612998963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,2047,0.021508799493312837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,2047,0.022811199724674224
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,4095,0.038940799236297605
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,4095,0.02956160008907318
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,8191,0.06240000128746033
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,8191,0.04544639885425568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,16383,0.10765759944915772
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,16383,0.07331680059432984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,1,0.012700800597667695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,32767,0.12639360427856444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,32767,0.19173280000686646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,3,0.012569600343704223
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,1,0.013348799943923951
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,3,0.013264000415802002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,15,0.012604799866676331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,7,0.013300800323486328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,7,0.012470400333404541
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,fp8,65535,0.2260335922241211
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,15,0.013369600474834441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,1,128,1,float16,float16,65535,0.39384000301361083
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,31,0.01268479973077774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,31,0.013460800051689148
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,63,0.012731200456619263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,127,0.012617599964141846
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,255,0.014440000057220459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,63,0.013398399949073792
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,255,0.015148800611495972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,511,0.0175135999917984
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,511,0.01894560009241104
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,127,0.01579200029373169
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,1023,0.02187040001153946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,1023,0.023020799458026885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,2047,0.038998401165008544
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,2047,0.028772801160812378
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,4095,0.050449597835540774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,4095,0.059943997859954835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,8191,0.1047935962677002
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,8191,0.0704255998134613
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,16383,0.11892800331115723
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,16383,0.18848320245742797
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,1,0.01135680004954338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,1,0.012160000205039979
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,32767,0.2219248056411743
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,3,0.011400000005960465
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,3,0.01218239963054657
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,32767,0.36492319107055665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,7,0.012031999975442886
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,15,0.011521600186824799
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,7,0.011459200084209442
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,31,0.011435200273990632
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,31,0.012227199971675873
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,15,0.01916320025920868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,63,0.011495999991893768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,63,0.012174399942159653
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,127,0.011508800089359283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,127,0.01924159973859787
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,255,0.013171200454235078
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,fp8,65535,0.4173759937286377
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,255,0.01414559930562973
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,4,2,128,1,float16,float16,65535,0.6949776172637939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,511,0.01600479930639267
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,1023,0.01681919991970062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,511,0.029080000519752503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,2047,0.01669439971446991
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,4095,0.018199999630451203
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,1023,0.017903999984264375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,4095,0.027296000719070436
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,8191,0.02022400051355362
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,2047,0.01797119975090027
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,8191,0.019841599464416503
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,16383,0.027225598692893982
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,16383,0.04307680130004883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,32767,0.04259999990463257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,32767,0.03316639959812164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,1,0.017080000042915343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,float16,65535,0.0642527997493744
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,1,0.012203200161457062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,1,128,1,float16,fp8,65535,0.04877119958400726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,3,0.01695519983768463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,3,0.01175519973039627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,7,0.011547199636697768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,7,0.012191999703645706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,15,0.016972799599170686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,31,0.011854399740695954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,15,0.011579199880361556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,31,0.012305600196123123
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,63,0.011558400094509124
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,63,0.016935999691486358
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,127,0.011753600090742111
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,127,0.01239679977297783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,255,0.013225600123405457
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,255,0.014212800562381745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,511,0.02268799990415573
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,1023,0.017080000042915343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,511,0.01809920072555542
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,1023,0.018059200048446654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,2047,0.017057600617408752
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,2047,0.02354239970445633
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,4095,0.01903039962053299
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,4095,0.018486399948596955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,8191,0.024560000002384185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,8191,0.023747199773788454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,16383,0.041515201330184937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,16383,0.030558401346206666
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,32767,0.06162559986114502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,32767,0.0458976000547409
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,1,0.015294399857521058
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,1,0.015931199491024017
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,3,0.016025599837303162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,float16,65535,0.10470720529556274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,3,0.014894400537014008
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,4,2,128,1,float16,fp8,65535,0.08543360233306885
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,7,0.015315200388431548
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,7,0.01587039977312088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,15,0.015854400396347047
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,15,0.014996799826622009
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,31,0.0173567995429039
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,31,0.016033600270748138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,63,0.01520639955997467
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,63,0.01576319932937622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,127,0.014873600006103516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,127,0.018012799322605133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,255,0.017636799812316896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,255,0.017518399655818938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,511,0.02133760005235672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,511,0.02144159972667694
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,1023,0.040380799770355226
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,1023,0.029523199796676634
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,2047,0.060288000106811526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,2047,0.04268800020217896
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,4095,0.10261119604110717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,4095,0.07776960134506225
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,8191,0.18738720417022706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,8191,0.11906880140304565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,1,0.018335999548435213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,16383,0.35663840770721433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,16383,0.2175823926925659
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,1,0.02006720006465912
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,7,0.018438400328159334
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,3,0.01849119961261749
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,3,0.020024000108242034
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,7,0.019896000623703003
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,15,0.01823839992284775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,15,0.020139199495315552
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,31,0.018404799699783325
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,63,0.018296000361442567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,fp8,32767,0.4195280075073242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,31,0.020044800639152528
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,63,0.020183999836444855
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,127,0.01857600063085556
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,1,128,1,float16,float16,32767,0.8410991668701172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,127,0.019705599546432494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,255,0.02404319941997528
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,511,0.03882080018520355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,511,0.03197759985923767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,255,0.02245599925518036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,1023,0.047068798542022706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,1023,0.07695519924163818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,2047,0.1050976037979126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,2047,0.0721776008605957
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,4095,0.12290879487991332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,4095,0.18823039531707764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,1,0.02072640061378479
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,1,0.022275200486183165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,8191,0.3646048069000244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,8191,0.3559839963912964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,3,0.0346127986907959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,3,0.022116799652576447
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,16383,0.41339998245239257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,7,0.03579359948635101
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,7,0.022870400547981264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,16383,0.6899968147277832
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,15,0.03824000060558319
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,31,0.02147199958562851
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,15,0.021091200411319733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,63,0.031215998530387878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,31,0.02256480008363724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,63,0.02297919988632202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,127,0.021247999370098115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,127,0.033855998516082765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,255,0.027059200406074523
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,255,0.024579200148582458
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,511,0.04258080124855042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,511,0.04917280077934265
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,fp8,32767,0.7996208190917968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,1023,0.050121599435806276
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,1023,0.06583679914474487
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,2047,0.10779520273208618
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,2047,0.09358400106430054
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,4,2,128,1,float16,float16,32767,1.3607040405273438
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,1,0.036740800738334654
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,4095,0.12517919540405273
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,3,0.03681919872760773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,4095,0.21185760498046874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,3,0.031350401043891904
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,fp8,8191,0.2232032060623169
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,1,128,1,float16,float16,8191,0.3584847927093506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,1,0.03145439922809601
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,7,0.032767999172210696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,15,0.028094398975372314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,7,0.031240001320838928
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,31,0.03149920105934143
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,63,0.0328575998544693
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,15,0.03110080063343048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,63,0.03134239912033081
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,31,0.02863999903202057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,127,0.029177600145339967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,255,0.04912959933280945
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,255,0.03954559862613678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,511,0.06707040071487427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,127,0.03160800039768219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,511,0.05689120292663574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,1023,0.1121440052986145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,1023,0.08708800077438354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,2047,0.19342559576034546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,2047,0.12799839973449706
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,1,0.034238401055336
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,4095,0.23202240467071533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,3,0.037062400579452516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,1,0.03689759969711304
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,4095,0.3592623949050903
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,3,0.03397440016269684
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,7,0.03710399866104126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,15,0.03388000130653381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,7,0.034599998593330385
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,31,0.03389439880847931
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,31,0.03707360029220581
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,15,0.03731040060520172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,63,0.03394399881362915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,63,0.037648001313209535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,127,0.03434560000896454
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,127,0.037088000774383546
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,float16,8191,0.6934544086456299
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,255,0.05191519856452942
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,255,0.04565759897232056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,511,0.07345600128173828
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,4,2,128,1,float16,fp8,8191,0.7198304176330567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,511,0.06150559782981872
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,1023,0.11728639602661133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,1023,0.08928639888763427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,1,0.04812000095844269
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,2047,0.1373695969581604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,1,0.05386880040168762
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,3,0.04866879880428314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,3,0.053569602966308597
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,2047,0.20934720039367677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,15,0.0486160010099411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,7,0.048854398727416995
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,7,0.05393120050430298
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,float16,4095,0.36492960453033446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,15,0.05379520058631897
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,31,0.04886719882488251
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,31,0.054283201694488525
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,63,0.049955201148986814
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,63,0.05386880040168762
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,127,0.05655840039253235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,127,0.05486559867858887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,255,0.08040800094604492
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,255,0.07243040204048157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,511,0.12092800140380859
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,511,0.10205600261688233
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,1023,0.20620319843292237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,1023,0.15188319683074952
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,1,0.011724799871444702
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,1,0.012331199645996094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,2047,0.248307204246521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,3,0.01168000027537346
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,1,128,1,float16,fp8,4095,1.207801628112793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,2047,0.3667216062545776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,7,0.012443199753761292
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,3,0.012591999769210816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,15,0.012374400347471236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,7,0.05148800015449524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,31,0.011825600266456604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,15,0.01186719983816147
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,31,0.012388800084590913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,63,0.051836800575256345
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,fp8,4095,0.4412576198577881
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,63,0.012486399710178375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,127,0.01183520033955574
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,127,0.012406399846076966
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,255,0.014455999433994293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,4,2,128,1,float16,float16,4095,0.6985167980194091
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,255,0.06026880145072937
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,511,0.01810719966888428
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,1023,0.016868799924850464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,1023,0.018223999440670012
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,511,0.016200000047683717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,4095,0.019152000546455383
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,2047,0.08608480095863343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,4095,0.01851679980754852
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,8191,0.024451200664043427
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,2047,0.017504000663757326
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,8191,0.02375040054321289
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,16383,0.042828801274299624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,16383,0.12793439626693726
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,32767,0.06485440135002137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,65535,0.07568479776382446
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,fp8,32767,0.048414400219917296
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,1,128,1,float16,float16,65535,0.10796639919281006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,1,0.011825600266456604
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,1,0.030905601382255555
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,3,0.011681599915027619
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,15,0.011905600130558015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,3,0.012355200201272964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,15,0.030932798981666565
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,7,0.011923199892044068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,7,0.012654399871826172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,31,0.011710400134325028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,31,0.012680000066757202
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,63,0.011796800047159195
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,63,0.031006398797035217
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,127,0.011648000031709672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,127,0.012567999958992004
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,255,0.01348000019788742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,255,0.014715200662612915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,511,0.016728000342845918
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,511,0.0182559996843338
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,1023,0.027241599559783936
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,1023,0.018164800107479097
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,2047,0.018931199610233308
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,2047,0.018649600446224213
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,4095,0.023528000712394713
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,4095,0.022233599424362184
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,8191,0.02945919930934906
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,8191,0.051708799600601194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,16383,0.06063839793205261
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,16383,0.04454880058765411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,32767,0.10449759960174561
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,32767,0.0696287989616394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,1,0.060139197111129764
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,1,0.08625119924545288
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,float16,65535,0.1883952021598816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,3,0.05465919971466064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,4,2,128,1,float16,fp8,65535,0.12204159498214721
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,7,0.05498719811439514
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,3,0.06094880104064941
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,7,0.06021760106086731
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,15,0.07705600261688232
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,31,0.05447040200233459
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,15,0.06059039831161499
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,63,0.056086397171020506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,31,0.0603007972240448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,63,0.060843199491500854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,127,0.06214560270309448
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,127,0.08699520230293274
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,255,0.08130559921264649
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,255,0.07829599976539611
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,511,0.1225167989730835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,511,0.10940159559249878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,1,0.08296800255775452
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,1023,0.21195518970489502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,1023,0.1595855951309204
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,3,0.08277279734611512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,1,0.09447680115699768
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,3,0.09464160203933716
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,7,0.0830128014087677
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,7,0.09378880262374878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,15,0.08259999752044678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,fp8,2047,0.2539072036743164
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,15,0.0946832001209259
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,31,0.08320800065994263
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,1,128,1,float16,float16,2047,0.4042479991912842
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,63,0.08642560243606567
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,31,0.09496639966964722
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,63,0.09672799706459045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,127,0.0933344006538391
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,127,0.10106559991836547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,255,0.13651039600372314
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,511,0.2168207883834839
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,511,0.18961600065231324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,255,0.3706144094467163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,1,0.09448000192642211
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,1023,0.2854127883911133
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,1023,0.3745919942855835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,3,0.09347519874572754
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,3,0.10499520301818847
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,1,0.3647200107574463
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,7,0.09405279755592347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,7,0.1054960012435913
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,15,0.09384480118751526
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,float16,2047,0.7029888153076171
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,31,0.09456639885902404
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,4,2,128,1,float16,fp8,2047,0.47737441062927244
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,31,0.1059440016746521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,63,0.09972000122070312
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,15,0.29100799560546875
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,127,0.10587519407272339
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,63,0.11076799631118775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,255,0.1442960023880005
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,255,0.13949120044708252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,127,0.11428799629211425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,511,0.20407838821411134
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,511,0.3604432106018066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,1,0.14777760505676268
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,1,0.17136000394821166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,3,0.1493872046470642
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,float16,1023,0.383078408241272
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,1,128,1,float16,fp8,1023,0.30329599380493166
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,7,0.147544002532959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,7,0.17140959501266478
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,15,0.14922239780426025
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,3,0.31818881034851076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,31,0.1500623941421509
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,63,0.15710400342941283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,15,0.17199679613113403
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,31,0.17093119621276856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,63,0.18255360126495362
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,127,0.2880064010620117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,127,0.18367040157318115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,255,0.24263200759887696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,255,0.23840160369873048
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,1,0.012006399780511856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,1,0.012744000554084778
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,511,0.39958078861236573
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,3,0.011854399740695954
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,3,0.01257600039243698
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,511,0.36077759265899656
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,7,0.011990399658679962
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,15,0.011870399862527848
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,15,0.012531200051307678
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,31,0.01276959925889969
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,63,0.011760000139474869
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,63,0.012675200402736665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,7,0.01252799928188324
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,31,0.011947199702262878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,127,0.012220799922943115
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,127,0.012668800354003907
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,255,0.013692800700664521
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,255,0.014744000136852264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,511,0.01661120057106018
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,511,0.01833920031785965
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,1023,0.017478400468826295
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,1023,0.018139199912548067
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,2047,0.018939200043678283
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,2047,0.018807999789714813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,fp8,1023,0.5518176078796386
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,4095,0.022763200104236603
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,4095,0.022431999444961548
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,8191,0.03994399905204773
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,8191,0.030195200443267824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,16383,0.06420000195503235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,32767,0.07373759746551514
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,16383,0.04726879894733429
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,32767,0.107259202003479
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,float16,65535,0.19160159826278686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,1,128,1,float16,fp8,65535,0.12639520168304444
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,1,0.012099199742078782
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,3,0.012111999839544297
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,1,0.012796799838542938
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,3,0.012652799487113953
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,15,0.012092799693346024
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,7,0.01197120025753975
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,31,0.012107200175523757
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,15,0.012956799566745758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,7,0.013079999387264252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,31,0.012695999443531036
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,63,0.012254399806261062
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,63,0.012716799974441528
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,127,0.012272000312805176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,127,0.013030399382114411
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,255,0.013977600634098053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,511,0.016916799545288085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,255,0.014833599328994751
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,511,0.018411199748516082
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,1023,0.01892800033092499
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,2047,0.022075200080871583
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,1023,0.018705600500106813
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,4,2,128,1,float16,float16,1023,1.63919677734375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,2047,0.02223999947309494
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,4095,0.038412800431251524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,4095,0.029846400022506714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,8191,0.0608672022819519
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,16383,0.06983199715614319
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,16383,0.10366079807281495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,32767,0.12250720262527466
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,32767,0.1884735941886902
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,1,0.013305599987506866
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,8191,0.20798399448394775
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,1,0.013977600634098053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,3,0.014060799777507783
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,3,0.013201600313186646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,7,0.013969600200653076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,float16,65535,0.357476806640625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,15,0.013332800567150116
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,15,0.013836799561977387
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,7,0.06055520176887512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,4,2,128,1,float16,fp8,65535,0.21982240676879883
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,31,0.013300800323486328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,31,0.014219200611114502
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,127,0.04456480145454407
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,255,0.0151296004652977
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,63,0.01404159963130951
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,255,0.016260799765586854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,63,0.0134320005774498
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,127,0.014193600416183472
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,511,0.019644799828529357
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,511,0.062224000692367554
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,1023,0.022491200268268584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,1023,0.02364159971475601
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,2047,0.03998399972915649
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,2047,0.02972959876060486
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,4095,0.06076800227165222
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,8191,0.07123200297355652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,8191,0.10502879619598389
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,4095,0.15229760408401488
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,16383,0.19303519725799562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,16383,0.12500640153884887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,1,0.01143999993801117
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,1,0.012225600332021714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,32767,0.36370880603790284
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,32767,0.22624640464782714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,3,0.011659199744462967
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,7,0.012195199728012085
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,7,0.011580800265073776
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,3,0.012276799976825714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,15,0.011507199704647064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,15,0.012206400185823441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,31,0.011667200177907944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,31,0.012323199957609176
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,65535,0.7006464004516602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,63,0.01156800016760826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,63,0.012276799976825714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,127,0.011456000059843064
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,255,0.013105599582195282
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,255,0.014215999841690063
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,511,0.016113600134849547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,127,0.012300799787044524
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,511,0.018177600204944612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,1023,0.016622400283813475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,1023,0.01791200041770935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,2047,0.018025599420070648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,4095,0.016804799437522888
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,4095,0.01812320053577423
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,2047,0.016921600699424742
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,8191,0.018900799751281738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,8191,0.02008959949016571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,16383,0.022276799380779266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,16383,0.02327679991722107
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,32767,0.024054400622844696
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,131071,0.8172528266906738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,32767,0.023953600227832793
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,65535,0.02802239954471588
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,65535,0.02815360128879547
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,float16,131071,0.0474128007888794
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,1,0.011383999884128571
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,3,0.01165120005607605
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,1,0.012359999865293504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,fp8,65535,1.4551936149597169
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,7,0.011470399796962738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,3,0.012222400307655335
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1,1,2,1,128,1,float16,fp8,131071,0.036313599348068236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,7,0.01223360002040863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,64,1,2,1,128,1,float16,float16,131071,1.3749872207641602
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,15,0.012161599844694138
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,31,0.011671999841928482
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,31,0.012374400347471236
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,63,0.012097600102424621
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,63,0.011692799627780914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,15,0.04368639886379242
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,127,0.012263999879360199
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,255,0.013158400356769562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,511,0.016025599837303162
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,255,0.01422400027513504
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,511,0.018352000415325163
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,1023,0.018068799376487733
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,127,0.011896000057458878
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,1023,0.06561599969863892
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,2047,0.017153599858283998
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,2047,0.018147200345993042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,4095,0.017427200078964235
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,4095,0.018216000497341157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,8191,0.07480480074882508
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,16383,0.02425280064344406
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,8191,0.020815999805927278
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,16383,0.023758399486541747
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,32767,0.027534401416778563
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,32767,0.027447998523712158
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,65535,0.033983999490737916
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,1,0.013300800323486328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,1,0.012753599882125854
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,65535,0.11867359876632691
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,fp8,131071,0.0507423996925354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,3,0.01252640038728714
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,3,0.013177600502967835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2,1,2,1,128,1,float16,float16,131071,0.06658080220222473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,7,0.012857599556446076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,7,0.029499199986457825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,15,0.012451200187206269
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,15,0.013128000497817992
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,31,0.012545600533485413
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,63,0.012857599556446076
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,31,0.013705599308013915
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,63,0.029897600412368774
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,127,0.01340479999780655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,127,0.012591999769210816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,255,0.014407999813556671
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,255,0.015689599514007568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,511,0.01759999990463257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,511,0.042763200402259824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,1023,0.01956160068511963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,1023,0.019361600279808044
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,2047,0.02197439968585968
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,2047,0.023180800676345825
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,4095,0.03949120044708252
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,4095,0.04620000123977661
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,8191,0.06219040155410767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,8191,0.04581120014190674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,16383,0.1083791971206665
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,16383,0.07364320158958435
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,32767,0.19309439659118652
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,32767,0.1916159987449646
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,1,0.011750400066375732
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,1,0.019324800372123717
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,65535,0.22611041069030763
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,3,0.011793600022792816
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,65535,0.3615504026412964
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,3,0.020425599813461304
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,7,0.01247360035777092
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,7,0.011635199934244157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,15,0.02111999988555908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,31,0.011767999827861786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,15,0.011767999827861786
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,31,0.012316799908876418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,63,0.0200095996260643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,63,0.012408000230789185
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,127,0.01162400022149086
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,127,0.012368000298738479
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,255,0.022536000609397887
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,255,0.01435679942369461
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,fp8,131071,0.4165775775909424
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,511,0.016257600486278535
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,511,0.01820479929447174
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,32,1,2,1,128,1,float16,float16,131071,0.6986176013946533
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,1023,0.029100799560546876
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,1023,0.018147200345993042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,2047,0.018415999412536622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,4095,0.01838880032300949
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,4095,0.01738079935312271
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,8191,0.035236799716949464
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,2047,0.01690399944782257
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,16383,0.02797439992427826
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,8191,0.020710399746894835
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,16383,0.02740960121154785
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,32767,0.042972800135612485
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,32767,0.03388000130653381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,65535,0.07596160173416137
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,1,0.014873600006103516
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,65535,0.04970720112323761
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,1,0.015574400126934052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,3,0.015255999565124512
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,fp8,131071,0.07684959769248963
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,4,1,2,1,128,1,float16,float16,131071,0.1093440055847168
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,7,0.014804799854755402
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,3,0.022968000173568724
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,7,0.015945599973201753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,15,0.0160303995013237
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,15,0.014681600034236908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,31,0.014830400049686433
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,63,0.015038399398326874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,63,0.02282720059156418
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,127,0.015462400019168853
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,31,0.015825599431991577
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,127,0.015542399883270264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,255,0.016527999937534333
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,255,0.018087999522686006
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,511,0.027595201134681703
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,511,0.021524800360202788
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,1023,0.03862560093402863
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,1023,0.029841598868370057
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,2047,0.05977920293807983
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,2047,0.04296320080757141
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,4095,0.0681007981300354
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,4095,0.10849119424819946
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,8191,0.12050080299377441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,8191,0.18686720132827758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,16383,0.2190256118774414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,16383,0.35743839740753175
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,1,0.020798400044441223
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,1,0.02258400022983551
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,3,0.02104640007019043
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,3,0.022543999552726745
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,7,0.021048000454902648
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,32767,0.6927519798278808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,7,0.022491200268268584
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,15,0.020899200439453126
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,31,0.020980800688266753
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,31,0.022470399737358093
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,15,0.0226623997092247
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,32767,0.7776879787445068
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,63,0.021028800308704375
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,63,0.022697600722312927
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,127,0.021142399311065672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,255,0.024646399915218352
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,255,0.026980799436569215
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,127,0.06352480053901673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,511,0.03454880118370056
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,511,0.04184800088405609
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,fp8,65535,0.8054719924926758
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,1023,0.06616799831390381
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,1023,0.14183839559555053
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,2047,0.0735696017742157
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,2047,0.10755200386047363
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,4095,0.12533600330352784
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,128,1,2,1,128,1,float16,float16,65535,1.3700688362121582
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,4095,0.21916160583496094
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,1,0.03451200127601624
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,1,0.03743039965629578
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,8191,0.22401280403137208
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,3,0.037380799651145935
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,3,0.03452959954738617
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,7,0.033888000249862674
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,7,0.037457600235939026
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,15,0.037294399738311765
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,fp8,16383,0.42301440238952637
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,15,0.03478400111198425
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,31,0.03444640040397644
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,8191,0.6625423908233643
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,31,0.03739359974861145
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,256,1,2,1,128,1,float16,float16,16383,0.695366382598877
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,63,0.03758240044116974
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,127,0.03475840091705322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,63,0.03457919955253601
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,127,0.037596800923347475
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,255,0.0455951988697052
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,255,0.19423840045928956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,511,0.0783631980419159
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,511,0.06304799914360046
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,1023,0.11900800466537476
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,1023,0.08974559903144837
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,2047,0.13573119640350342
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,2047,0.1989680051803589
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,1,0.011878400295972823
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,3,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,1,0.012436799705028534
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,3,0.011739200353622437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,4095,0.23559200763702393
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,7,0.011777599900960922
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,7,0.012513600289821625
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,15,0.01180799975991249
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,31,0.011817599833011627
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,31,0.012508800625801087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,63,0.011856000125408172
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,15,0.012750400602817536
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,63,0.012670400738716125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,127,0.011790399998426437
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,127,0.012574400007724761
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,255,0.013281600177288055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,255,0.014505599439144135
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,511,0.01646080017089844
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,fp8,8191,0.438478422164917
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,511,0.01823520064353943
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,1023,0.017239999771118165
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,1023,0.018592000007629395
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,2047,0.01756319999694824
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,8191,0.697599983215332
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,4095,0.01907840073108673
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,4095,0.019177600741386414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,8191,0.024238400161266327
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,8191,0.024377599358558655
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,2047,0.018729600310325622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,512,1,2,1,128,1,float16,float16,4095,0.9136927604675293
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,16383,0.03403840065002441
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,16383,0.04664320051670075
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,32767,0.06508799791336059
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,32767,0.04994080066680908
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,65535,0.07763839960098266
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,65535,0.10960960388183594
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,1,0.054504001140594484
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,fp8,131071,0.13015999794006347
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,1,0.06098880171775818
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,3,0.05522400140762329
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,3,0.06097279787063599
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,7,0.05452479720115662
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,15,0.05542399883270264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,7,0.06066719889640808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,15,0.061268800497055055
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,31,0.055379199981689456
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,31,0.060524797439575194
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,63,0.05599200129508972
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,63,0.061003202199935914
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,127,0.06231200098991394
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,127,0.06219040155410767
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,8,1,2,1,128,1,float16,float16,131071,0.6199600219726562
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,255,0.08464000225067139
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,255,0.07908160090446473
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,511,0.12559679746627808
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,511,0.11012959480285645
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,1023,0.21056320667266845
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,1,0.09521920084953309
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,2047,0.2585376024246216
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,2047,0.3746239900588989
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,1,0.10673919916152955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,3,0.09485920071601868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,3,0.10649919509887695
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,1023,0.6612063884735108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,7,0.09426079988479615
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,7,0.10686240196228028
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,fp8,4095,0.4560527801513672
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,15,0.09489279985427856
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,1024,1,2,1,128,1,float16,float16,4095,0.7042128086090088
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,15,0.10698080062866211
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,31,0.10641759634017944
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,63,0.10118080377578735
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,127,0.10602719783782959
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,31,0.3166896104812622
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,63,0.1083184003829956
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,255,0.14426239728927612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,127,0.11612000465393066
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,255,0.14142719507217408
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,511,0.21755518913269042
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,1,0.011931200325489045
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,511,0.2051919937133789
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,1,0.012839999794960023
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,3,0.01188960000872612
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,7,0.012167999893426895
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,1023,0.3062959909439087
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,7,0.01263359934091568
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,3,0.012943999469280243
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,15,0.012135999649763108
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,31,0.011806400120258331
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,31,0.013009600341320038
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,63,0.012049599736928939
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,15,0.012468799948692322
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,63,0.012582400441169738
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,127,0.012055999785661697
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,127,0.013022400438785553
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,255,0.013631999492645264
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,255,0.014555199444293976
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,511,0.01682240068912506
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,511,0.018430399894714355
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,fp8,2047,0.49108638763427737
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,1023,0.667852783203125
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,1023,0.01765599995851517
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,1023,0.018961599469184874
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,2048,1,2,1,128,1,float16,float16,2047,0.7121871948242188
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,2047,0.01939679980278015
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,2047,0.018780800700187682
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,8191,0.03087199926376343
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,4095,0.02284640073776245
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,8191,0.040273600816726686
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,4095,0.10599679946899414
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,16383,0.04880160093307495
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,16383,0.06980320215225219
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,32767,0.10736320018768311
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,32767,0.07471680045127868
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,65535,0.12640320062637328
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,65535,0.1923840045928955
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,fp8,131071,0.22511680126190187
SGLang,0.5.8.post1,NVIDIA H100 80GB HBM3,generation_attention,flash_attention,16,1,2,1,128,1,float16,float16,131071,0.8262543678283691
