framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,2,128,1,fp8,fp8,0,76.70477905273438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,2,128,1,bfloat16,fp8,0,77.75586547851563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,1,128,1,bfloat16,bfloat16,0,91.53700561523438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,1,128,1,bfloat16,fp8,0,79.27929077148437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,1,128,1,fp8,fp8,0,76.9113037109375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,4,128,1,bfloat16,fp8,0,77.13491821289062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,2,128,1,bfloat16,bfloat16,0,92.62169189453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,4,128,1,bfloat16,bfloat16,0,92.09097290039062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,4,128,1,fp8,fp8,0,76.61715087890624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,8,128,1,bfloat16,fp8,0,77.73029174804688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,1,128,1,bfloat16,bfloat16,0,45.71747741699219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,8,128,1,fp8,fp8,0,79.90698852539063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,1,128,1,bfloat16,fp8,0,38.11539916992187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,1,128,1,fp8,fp8,0,38.538351440429686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,2,128,1,bfloat16,bfloat16,0,45.72013549804687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,2,128,1,bfloat16,fp8,0,39.68836975097656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,2,128,1,fp8,fp8,0,38.146087646484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,4,128,1,bfloat16,bfloat16,0,45.57928466796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,4,128,1,bfloat16,fp8,0,38.71171569824219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,4,128,1,fp8,fp8,0,39.75483703613281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,8,128,1,bfloat16,fp8,0,38.80010986328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,8,128,1,fp8,fp8,0,38.87504272460937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,8,128,1,bfloat16,bfloat16,0,45.7834228515625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,96,128,1,bfloat16,bfloat16,0,23.64606475830078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,96,128,1,bfloat16,fp8,0,19.16059265136719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,96,128,1,fp8,fp8,0,20.27533416748047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,1,128,1,bfloat16,bfloat16,0,21.652680969238283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,96,128,1,bfloat16,fp8,0,39.819873046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,96,128,1,fp8,fp8,0,39.964288330078126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,1,128,1,bfloat16,fp8,0,20.49401397705078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,1,128,1,fp8,fp8,0,20.113433837890625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,96,96,128,1,bfloat16,bfloat16,0,48.610592651367185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,2,128,1,bfloat16,bfloat16,0,22.887232971191406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,2,128,1,bfloat16,fp8,0,19.775350952148436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,96,8,128,1,bfloat16,bfloat16,0,93.17864990234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,2,128,1,fp8,fp8,0,18.547090148925783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,4,128,1,bfloat16,fp8,0,19.151692199707032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,4,128,1,fp8,fp8,0,18.724580383300783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,4,128,1,bfloat16,bfloat16,0,22.356747436523438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,96,128,1,bfloat16,bfloat16,0,11.937267303466797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,8,128,1,bfloat16,bfloat16,0,22.519154357910157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,8,128,1,bfloat16,fp8,0,19.28241882324219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,96,128,1,bfloat16,fp8,0,9.814545440673829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,96,128,1,fp8,fp8,0,10.536656188964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,96,8,128,1,fp8,fp8,0,18.72081756591797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,1,128,1,bfloat16,bfloat16,0,11.519950103759765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,1,128,1,bfloat16,fp8,0,9.747716522216797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,1,128,1,fp8,fp8,0,9.81378402709961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,2,128,1,bfloat16,fp8,0,9.672459411621094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,2,128,1,bfloat16,bfloat16,0,11.764726257324218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,2,128,1,fp8,fp8,0,10.252065277099609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,4,128,1,bfloat16,fp8,0,9.937654113769531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,4,128,1,fp8,fp8,0,10.030168151855468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,4,128,1,bfloat16,bfloat16,0,12.013297271728515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,8,128,1,bfloat16,bfloat16,0,11.9279296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,8,128,1,bfloat16,fp8,0,9.798214721679688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,96,8,128,1,fp8,fp8,0,9.936465454101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,1,128,1,bfloat16,fp8,0,44.006265258789064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,1,128,1,fp8,fp8,0,43.29148864746094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,2,128,1,bfloat16,fp8,0,45.098175048828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,2,128,1,fp8,fp8,0,44.070623779296874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,1,128,1,bfloat16,bfloat16,0,51.85269775390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,4,128,1,bfloat16,fp8,0,44.36692810058594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,2,128,1,bfloat16,bfloat16,0,51.5982177734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,4,128,1,bfloat16,bfloat16,0,52.47163696289063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,96,128,1,bfloat16,fp8,0,23.07496032714844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,96,128,1,fp8,fp8,0,22.98826904296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,96,128,1,bfloat16,bfloat16,0,28.343853759765626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,1,128,1,bfloat16,bfloat16,0,26.297467041015626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,4,128,1,fp8,fp8,0,44.20624694824219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,8,128,1,fp8,fp8,0,44.359808349609374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,8,128,1,bfloat16,fp8,0,45.276171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,96,8,128,1,bfloat16,bfloat16,0,51.728179931640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,1,128,1,bfloat16,fp8,0,22.030580139160158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,1,128,1,fp8,fp8,0,21.81464385986328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,2,128,1,bfloat16,bfloat16,0,25.791439819335938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,2,128,1,bfloat16,fp8,0,22.445562744140624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,2,128,1,fp8,fp8,0,22.161341857910156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,4,128,1,bfloat16,fp8,0,22.822154235839843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,4,128,1,fp8,fp8,0,21.777323913574218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,4,128,1,bfloat16,bfloat16,0,26.630322265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,96,128,1,bfloat16,fp8,0,11.476299285888672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,8,128,1,bfloat16,fp8,0,22.393174743652345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,96,128,1,bfloat16,bfloat16,0,14.454426574707032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,8,128,1,bfloat16,bfloat16,0,26.58411560058594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,96,8,128,1,fp8,fp8,0,21.865803527832032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,96,128,1,fp8,fp8,0,11.62597427368164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,1,128,1,bfloat16,fp8,0,10.524686431884765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,1,128,1,fp8,fp8,0,10.581655883789063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,1,128,1,bfloat16,bfloat16,0,12.930990600585938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,2,128,1,bfloat16,fp8,0,11.101910400390626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,2,128,1,fp8,fp8,0,10.720642852783204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,2,128,1,bfloat16,bfloat16,0,12.935478210449219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,4,128,1,bfloat16,fp8,0,11.05771713256836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,4,128,1,bfloat16,bfloat16,0,13.396514892578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,96,128,1,bfloat16,fp8,0,5.632577514648437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,4,128,1,fp8,fp8,0,10.877188873291015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,96,128,1,bfloat16,bfloat16,0,7.143910217285156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,8,128,1,bfloat16,fp8,0,10.675341033935547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,8,128,1,bfloat16,bfloat16,0,13.433311462402344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,96,8,128,1,fp8,fp8,0,11.333102416992187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,96,128,1,fp8,fp8,0,5.882440185546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,1,128,1,bfloat16,fp8,0,5.3725135803222654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,1,128,1,bfloat16,bfloat16,0,6.6251373291015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,1,128,1,fp8,fp8,0,5.5470222473144535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,2,128,1,bfloat16,fp8,0,5.323926544189453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,2,128,1,bfloat16,bfloat16,0,6.449046325683594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,2,128,1,fp8,fp8,0,5.575518417358398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,4,128,1,bfloat16,fp8,0,5.476440048217773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,4,128,1,bfloat16,bfloat16,0,6.783042907714844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,4,128,1,fp8,fp8,0,5.44127197265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,8,128,1,bfloat16,fp8,0,5.459916687011718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,8,128,1,bfloat16,bfloat16,0,6.745464324951172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,96,8,128,1,fp8,fp8,0,5.558811187744141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,1,128,1,bfloat16,fp8,0,30.819638061523438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,2,128,1,bfloat16,fp8,0,29.592218017578126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,1,128,1,fp8,fp8,0,31.079403686523438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,2,128,1,fp8,fp8,0,30.607510375976563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,4,128,1,bfloat16,fp8,0,30.926229858398436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,1,128,1,bfloat16,bfloat16,0,36.900021362304685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,2,128,1,bfloat16,bfloat16,0,36.609466552734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,4,128,1,bfloat16,bfloat16,0,36.42008666992187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,96,128,1,bfloat16,fp8,0,16.220799255371094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,96,128,1,bfloat16,bfloat16,0,19.88016357421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,96,128,1,fp8,fp8,0,15.815078735351562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,1,128,1,bfloat16,bfloat16,0,18.058995056152344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,4,128,1,fp8,fp8,0,31.0070068359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,8,128,1,fp8,fp8,0,30.827413940429686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,8,128,1,bfloat16,fp8,0,31.384735107421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,96,8,128,1,bfloat16,bfloat16,0,35.90263061523437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,1,128,1,fp8,fp8,0,15.219989013671874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,1,128,1,bfloat16,fp8,0,15.795700073242188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,2,128,1,bfloat16,bfloat16,0,18.847372436523436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,2,128,1,bfloat16,fp8,0,15.240126037597657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,2,128,1,fp8,fp8,0,15.383755493164063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,4,128,1,bfloat16,fp8,0,16.05292205810547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,4,128,1,bfloat16,bfloat16,0,18.2467529296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,96,128,1,bfloat16,bfloat16,0,10.53231201171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,4,128,1,fp8,fp8,0,16.109414672851564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,8,128,1,bfloat16,fp8,0,16.558302307128905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,96,128,1,bfloat16,fp8,0,8.089292907714844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,8,128,1,bfloat16,bfloat16,0,18.529261779785156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,96,8,128,1,fp8,fp8,0,14.944444274902343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,96,128,1,fp8,fp8,0,8.190782165527343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,1,128,1,bfloat16,bfloat16,0,9.024912261962891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,1,128,1,bfloat16,fp8,0,7.407004547119141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,1,128,1,fp8,fp8,0,7.421315002441406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,2,128,1,bfloat16,fp8,0,7.722150421142578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,2,128,1,bfloat16,bfloat16,0,9.390599822998047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,2,128,1,fp8,fp8,0,7.619089508056641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,4,128,1,bfloat16,fp8,0,7.719963073730469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,4,128,1,bfloat16,bfloat16,0,9.378600311279296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,4,128,1,fp8,fp8,0,7.448398590087891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,96,128,1,bfloat16,fp8,0,3.994113540649414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,8,128,1,bfloat16,fp8,0,7.446797180175781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,8,128,1,bfloat16,bfloat16,0,9.1303955078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,96,128,1,bfloat16,bfloat16,0,4.898852920532226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,96,128,1,fp8,fp8,0,4.104390335083008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,1,128,1,bfloat16,bfloat16,0,4.544508743286133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,96,8,128,1,fp8,fp8,0,7.763246154785156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,1,128,1,bfloat16,fp8,0,3.6875728607177733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,1,128,1,fp8,fp8,0,3.789748764038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,2,128,1,bfloat16,fp8,0,3.757215881347656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,2,128,1,bfloat16,bfloat16,0,4.423097610473633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,2,128,1,fp8,fp8,0,3.9052734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,4,128,1,bfloat16,bfloat16,0,4.603219223022461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,4,128,1,bfloat16,fp8,0,3.907668685913086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,4,128,1,fp8,fp8,0,3.7783840179443358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,8,128,1,bfloat16,bfloat16,0,4.568622589111328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,8,128,1,bfloat16,fp8,0,3.8171665191650392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,96,8,128,1,fp8,fp8,0,3.7686832427978514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,2,128,1,bfloat16,fp8,0,40.2698486328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,1,128,1,fp8,fp8,0,40.826901245117185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,2,128,1,fp8,fp8,0,39.85117797851562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,4,128,1,bfloat16,fp8,0,40.12595825195312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,1,128,1,bfloat16,fp8,0,42.0523193359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,1,128,1,bfloat16,bfloat16,0,47.17835693359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,2,128,1,bfloat16,bfloat16,0,47.5775146484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,4,128,1,bfloat16,bfloat16,0,47.789666748046876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,96,128,1,bfloat16,fp8,0,20.952122497558594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,96,128,1,fp8,fp8,0,20.92937469482422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,96,128,1,bfloat16,bfloat16,0,25.26834716796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,1,128,1,bfloat16,bfloat16,0,23.691128540039063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,4,128,1,fp8,fp8,0,40.32087707519531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,8,128,1,bfloat16,fp8,0,40.00153503417969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,8,128,1,fp8,fp8,0,40.2287353515625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,1,128,1,bfloat16,fp8,0,20.462924194335937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,96,8,128,1,bfloat16,bfloat16,0,48.54390869140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,1,128,1,fp8,fp8,0,20.58667449951172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,2,128,1,bfloat16,fp8,0,20.2171142578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,2,128,1,bfloat16,bfloat16,0,24.88885955810547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,2,128,1,fp8,fp8,0,20.11144256591797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,4,128,1,bfloat16,fp8,0,20.145632934570312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,4,128,1,bfloat16,bfloat16,0,23.687353515625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,4,128,1,fp8,fp8,0,19.638937377929686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,96,128,1,bfloat16,bfloat16,0,12.815756225585938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,8,128,1,bfloat16,fp8,0,19.988304138183594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,96,128,1,bfloat16,fp8,0,10.717729949951172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,8,128,1,bfloat16,bfloat16,0,24.130001831054688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,96,128,1,fp8,fp8,0,10.592118072509766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,96,8,128,1,fp8,fp8,0,20.33563232421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,1,128,1,bfloat16,bfloat16,0,11.676262664794923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,1,128,1,bfloat16,fp8,0,10.071425628662109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,1,128,1,fp8,fp8,0,10.727067565917968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,2,128,1,bfloat16,fp8,0,9.938614654541016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,2,128,1,bfloat16,bfloat16,0,12.277196502685547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,2,128,1,fp8,fp8,0,10.173738861083985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,4,128,1,bfloat16,fp8,0,9.719737243652343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,4,128,1,bfloat16,bfloat16,0,12.271040344238282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,4,128,1,fp8,fp8,0,10.197831726074218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,96,128,1,bfloat16,bfloat16,0,6.443548583984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,96,128,1,bfloat16,fp8,0,5.417747116088867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,8,128,1,bfloat16,fp8,0,9.7442626953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,8,128,1,bfloat16,bfloat16,0,12.403108978271485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,96,128,1,fp8,fp8,0,5.239131164550781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,96,8,128,1,fp8,fp8,0,10.053813171386718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,1,128,1,bfloat16,bfloat16,0,5.9338623046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,1,128,1,bfloat16,fp8,0,4.85806884765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,1,128,1,fp8,fp8,0,4.849267196655274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,2,128,1,bfloat16,bfloat16,0,5.8772735595703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,2,128,1,bfloat16,fp8,0,4.815972900390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,2,128,1,fp8,fp8,0,5.146814346313477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,4,128,1,bfloat16,fp8,0,5.0090576171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,4,128,1,bfloat16,bfloat16,0,5.932075119018554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,4,128,1,fp8,fp8,0,5.058596801757813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,8,128,1,bfloat16,fp8,0,4.866007995605469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,96,128,1,bfloat16,bfloat16,0,3.183296012878418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,8,128,1,bfloat16,bfloat16,0,5.8082622528076175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,96,128,1,bfloat16,fp8,0,2.883644866943359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,96,128,1,fp8,fp8,0,2.6322383880615234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,1,128,1,bfloat16,fp8,0,2.4390512466430665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,1,128,1,bfloat16,bfloat16,0,3.065907287597656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,96,8,128,1,fp8,fp8,0,5.1626640319824215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,1,128,1,fp8,fp8,0,2.4134639739990233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,2,128,1,bfloat16,fp8,0,2.59674072265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,2,128,1,bfloat16,bfloat16,0,2.8853904724121096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,2,128,1,fp8,fp8,0,2.5333391189575196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,4,128,1,bfloat16,fp8,0,2.4623039245605467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,4,128,1,bfloat16,bfloat16,0,2.861591911315918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,4,128,1,fp8,fp8,0,2.4705631256103517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,8,128,1,bfloat16,bfloat16,0,2.67825927734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,8,128,1,bfloat16,fp8,0,2.4497663497924806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,96,8,128,1,fp8,fp8,0,2.438409614562988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,1,128,1,fp8,fp8,0,23.199351501464843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,2,128,1,bfloat16,fp8,0,22.78363494873047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,1,128,1,bfloat16,fp8,0,23.962184143066406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,2,128,1,fp8,fp8,0,23.559046936035156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,4,128,1,bfloat16,fp8,0,23.408174133300783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,1,128,1,bfloat16,bfloat16,0,28.254763793945312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,2,128,1,bfloat16,bfloat16,0,27.245516967773437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,4,128,1,bfloat16,bfloat16,0,27.853842163085936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,96,128,1,bfloat16,fp8,0,12.129964447021484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,96,128,1,fp8,fp8,0,12.340790557861329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,96,128,1,bfloat16,bfloat16,0,15.33499755859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,1,128,1,bfloat16,bfloat16,0,13.340371704101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,4,128,1,fp8,fp8,0,23.29996337890625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,8,128,1,bfloat16,fp8,0,23.852955627441407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,8,128,1,fp8,fp8,0,23.93958282470703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,96,8,128,1,bfloat16,bfloat16,0,27.487710571289064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,1,128,1,bfloat16,fp8,0,11.855073547363281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,1,128,1,fp8,fp8,0,11.464155578613282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,2,128,1,bfloat16,bfloat16,0,13.662628173828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,2,128,1,bfloat16,fp8,0,11.623894500732423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,2,128,1,fp8,fp8,0,11.705290985107421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,4,128,1,bfloat16,fp8,0,11.775724792480469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,4,128,1,fp8,fp8,0,11.17419662475586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,4,128,1,bfloat16,bfloat16,0,14.620440673828124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,96,128,1,bfloat16,bfloat16,0,8.057968139648438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,8,128,1,bfloat16,fp8,0,11.76903839111328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,96,128,1,bfloat16,fp8,0,6.446176147460937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,8,128,1,bfloat16,bfloat16,0,14.259100341796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,96,8,128,1,fp8,fp8,0,11.347675323486328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,96,128,1,fp8,fp8,0,6.330347061157227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,1,128,1,bfloat16,fp8,0,5.854827117919922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,1,128,1,fp8,fp8,0,5.602729415893554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,1,128,1,bfloat16,bfloat16,0,7.0955757141113285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,2,128,1,bfloat16,bfloat16,0,6.812582397460938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,2,128,1,bfloat16,fp8,0,5.836225509643555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,2,128,1,fp8,fp8,0,5.604929733276367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,4,128,1,bfloat16,bfloat16,0,7.198588562011719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,4,128,1,bfloat16,fp8,0,5.851512145996094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,4,128,1,fp8,fp8,0,6.164947128295898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,96,128,1,bfloat16,fp8,0,3.0350271224975587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,96,128,1,bfloat16,bfloat16,0,3.8330463409423827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,8,128,1,bfloat16,bfloat16,0,6.784977722167969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,8,128,1,bfloat16,fp8,0,5.93287353515625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,96,128,1,fp8,fp8,0,3.112214469909668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,1,128,1,bfloat16,bfloat16,0,3.366944122314453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,96,8,128,1,fp8,fp8,0,5.73353271484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,1,128,1,bfloat16,fp8,0,2.897444725036621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,1,128,1,fp8,fp8,0,2.7894655227661134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,2,128,1,fp8,fp8,0,2.7933231353759767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,2,128,1,bfloat16,fp8,0,2.854047966003418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,2,128,1,bfloat16,bfloat16,0,3.3335742950439453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,4,128,1,bfloat16,bfloat16,0,3.1760208129882814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,4,128,1,fp8,fp8,0,2.821356773376465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,4,128,1,bfloat16,fp8,0,3.155135917663574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,96,128,1,bfloat16,bfloat16,0,1.8201440811157226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,96,128,1,bfloat16,fp8,0,1.6407215118408203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,8,128,1,bfloat16,fp8,0,2.8177583694458006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,8,128,1,fp8,fp8,0,2.8218399047851563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,96,8,128,1,bfloat16,bfloat16,0,3.65338249206543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,1,128,1,bfloat16,bfloat16,0,1.555177593231201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,96,128,1,fp8,fp8,0,1.8944000244140624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,1,128,1,fp8,fp8,0,1.6214111328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,2,128,1,bfloat16,bfloat16,0,1.524608039855957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,2,128,1,bfloat16,fp8,0,1.4018848419189454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,2,128,1,fp8,fp8,0,1.5888575553894042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,4,128,1,bfloat16,fp8,0,1.4065360069274901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,4,128,1,bfloat16,bfloat16,0,1.6824224472045899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,4,128,1,fp8,fp8,0,1.4762160301208496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,8,128,1,bfloat16,bfloat16,0,1.634596824645996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,8,128,1,bfloat16,fp8,0,1.4154447555541991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,8,128,1,fp8,fp8,0,1.456828784942627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,96,1,128,1,bfloat16,fp8,0,1.4360671997070313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,1,128,1,bfloat16,fp8,0,21.27686309814453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,1,128,1,fp8,fp8,0,21.614459228515624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,2,128,1,bfloat16,fp8,0,21.202877807617188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,2,128,1,fp8,fp8,0,22.228904724121094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,4,128,1,bfloat16,fp8,0,21.036347961425783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,1,128,1,bfloat16,bfloat16,0,25.103411865234374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,2,128,1,bfloat16,bfloat16,0,25.692959594726563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,4,128,1,bfloat16,bfloat16,0,25.206207275390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,96,128,1,bfloat16,fp8,0,11.942316436767578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,96,128,1,fp8,fp8,0,12.091175842285157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,96,128,1,bfloat16,bfloat16,0,15.464886474609376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,1,128,1,bfloat16,bfloat16,0,13.988996887207032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,4,128,1,fp8,fp8,0,21.790164184570312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,8,128,1,bfloat16,fp8,0,21.514743041992187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,8,128,1,fp8,fp8,0,22.554762268066405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,96,8,128,1,bfloat16,bfloat16,0,25.9173095703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,1,128,1,bfloat16,fp8,0,10.626315307617187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,1,128,1,fp8,fp8,0,10.710590362548828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,2,128,1,bfloat16,fp8,0,10.642708587646485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,2,128,1,bfloat16,bfloat16,0,12.797590637207032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,2,128,1,fp8,fp8,0,10.6126220703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,4,128,1,bfloat16,bfloat16,0,12.635435485839844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,4,128,1,bfloat16,fp8,0,11.164603424072265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,4,128,1,fp8,fp8,0,10.648302459716797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,8,128,1,bfloat16,fp8,0,11.077732849121094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,96,128,1,bfloat16,bfloat16,0,7.121115112304688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,96,128,1,bfloat16,fp8,0,6.06116943359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,8,128,1,bfloat16,bfloat16,0,13.122479248046876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,96,8,128,1,fp8,fp8,0,10.716782379150391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,96,128,1,fp8,fp8,0,6.066104125976563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,1,128,1,bfloat16,fp8,0,5.361172866821289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,1,128,1,bfloat16,bfloat16,0,7.390509033203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,2,128,1,bfloat16,fp8,0,5.307720184326172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,1,128,1,fp8,fp8,0,5.417873764038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,2,128,1,fp8,fp8,0,5.439308929443359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,4,128,1,bfloat16,fp8,0,5.366366577148438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,4,128,1,bfloat16,bfloat16,0,6.511646270751953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,4,128,1,fp8,fp8,0,5.471566390991211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,2,128,1,bfloat16,bfloat16,0,6.214401626586914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,96,128,1,bfloat16,bfloat16,0,3.634374237060547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,8,128,1,bfloat16,bfloat16,0,6.228416061401367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,8,128,1,bfloat16,fp8,0,5.447639846801758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,96,8,128,1,fp8,fp8,0,5.451297760009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,96,128,1,bfloat16,fp8,0,2.946721649169922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,96,128,1,fp8,fp8,0,3.120676803588867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,1,128,1,bfloat16,bfloat16,0,3.0912288665771483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,1,128,1,bfloat16,fp8,0,2.6338031768798826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,1,128,1,fp8,fp8,0,2.826464080810547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,2,128,1,bfloat16,bfloat16,0,3.3092864990234374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,2,128,1,fp8,fp8,0,2.68624324798584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,2,128,1,bfloat16,fp8,0,2.7020847320556642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,4,128,1,bfloat16,bfloat16,0,3.2445537567138674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,4,128,1,bfloat16,fp8,0,2.810932731628418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,96,128,1,bfloat16,fp8,0,1.4886223793029785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,4,128,1,fp8,fp8,0,2.8289440155029295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,96,128,1,bfloat16,bfloat16,0,1.732975959777832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,8,128,1,bfloat16,bfloat16,0,2.993332862854004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,8,128,1,bfloat16,fp8,0,2.6568815231323244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,96,128,1,fp8,fp8,0,1.620257568359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,1,128,1,bfloat16,bfloat16,0,1.6119440078735352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,96,8,128,1,fp8,fp8,0,2.736435127258301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,1,128,1,bfloat16,fp8,0,1.3266655921936035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,1,128,1,fp8,fp8,0,1.3309663772583007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,2,128,1,bfloat16,bfloat16,0,1.57772798538208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,2,128,1,fp8,fp8,0,1.3191328048706055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,2,128,1,bfloat16,fp8,0,1.5242992401123048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,4,128,1,bfloat16,bfloat16,0,1.420417594909668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,4,128,1,bfloat16,fp8,0,1.628868865966797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,4,128,1,fp8,fp8,0,1.448020839691162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,96,128,1,bfloat16,fp8,0,0.7922544002532959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,8,128,1,bfloat16,fp8,0,1.3348544120788575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,96,128,1,fp8,fp8,0,0.8205760002136231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,8,128,1,bfloat16,bfloat16,0,1.4600576400756835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,96,8,128,1,fp8,fp8,0,1.4757807731628418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,1,128,1,bfloat16,bfloat16,0,0.8091839790344239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,1,128,1,bfloat16,fp8,0,0.6664944171905518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,1,128,1,fp8,fp8,0,0.6900479793548584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,2,128,1,bfloat16,bfloat16,0,0.745304012298584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,2,128,1,bfloat16,fp8,0,0.7012815952301026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,2,128,1,fp8,fp8,0,0.7383600234985351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,4,128,1,bfloat16,bfloat16,0,0.7395264148712158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,4,128,1,bfloat16,fp8,0,0.7833471775054932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,4,128,1,fp8,fp8,0,0.7903039932250977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,8,128,1,bfloat16,bfloat16,0,0.7324495792388916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,8,128,1,bfloat16,fp8,0,0.7311312198638916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,8,128,1,fp8,fp8,0,0.7119487762451172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,96,96,128,1,bfloat16,bfloat16,0,0.85415678024292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,1,128,1,bfloat16,fp8,0,12.67493896484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,1,128,1,fp8,fp8,0,12.595545959472656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,2,128,1,bfloat16,fp8,0,12.490940856933594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,2,128,1,fp8,fp8,0,12.439031982421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,1,128,1,bfloat16,bfloat16,0,14.78497314453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,2,128,1,bfloat16,bfloat16,0,15.557441711425781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,4,128,1,bfloat16,bfloat16,0,14.54320831298828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,4,128,1,bfloat16,fp8,0,12.506593322753906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,96,128,1,bfloat16,fp8,0,7.418385314941406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,96,128,1,bfloat16,bfloat16,0,8.862347412109376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,96,128,1,fp8,fp8,0,7.17541732788086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,4,128,1,fp8,fp8,0,12.971714782714844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,8,128,1,bfloat16,fp8,0,12.880595397949218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,8,128,1,bfloat16,bfloat16,0,15.152932739257812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,96,8,128,1,fp8,fp8,0,12.711811065673828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,1,128,1,bfloat16,bfloat16,0,7.237776184082032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,1,128,1,bfloat16,fp8,0,6.694942474365234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,1,128,1,fp8,fp8,0,6.30775032043457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,2,128,1,bfloat16,bfloat16,0,7.475297546386718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,2,128,1,bfloat16,fp8,0,6.309763336181641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,2,128,1,fp8,fp8,0,6.4013923645019535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,4,128,1,bfloat16,bfloat16,0,7.870123291015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,4,128,1,bfloat16,fp8,0,6.255286407470703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,4,128,1,fp8,fp8,0,6.295217514038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,96,128,1,bfloat16,fp8,0,3.6987823486328124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,96,128,1,bfloat16,bfloat16,0,4.542150497436523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,8,128,1,bfloat16,fp8,0,6.3501136779785154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,8,128,1,bfloat16,bfloat16,0,8.2002685546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,96,8,128,1,fp8,fp8,0,6.963536071777344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,1,128,1,bfloat16,fp8,0,3.102783966064453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,1,128,1,bfloat16,bfloat16,0,3.493707275390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,1,128,1,fp8,fp8,0,3.140291213989258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,96,128,1,fp8,fp8,0,3.82470703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,2,128,1,bfloat16,fp8,0,3.1139104843139647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,2,128,1,bfloat16,bfloat16,0,3.6352638244628905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,2,128,1,fp8,fp8,0,3.3030574798583983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,4,128,1,bfloat16,bfloat16,0,3.6843902587890627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,4,128,1,bfloat16,fp8,0,3.4123184204101564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,4,128,1,fp8,fp8,0,3.265012741088867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,96,128,1,bfloat16,bfloat16,0,2.1529167175292967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,8,128,1,bfloat16,fp8,0,3.2490718841552733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,8,128,1,fp8,fp8,0,3.1518911361694335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,96,8,128,1,bfloat16,bfloat16,0,3.8218017578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,96,128,1,bfloat16,fp8,0,1.8320751190185547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,1,128,1,bfloat16,fp8,0,1.5596912384033204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,96,128,1,fp8,fp8,0,2.0513696670532227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,1,128,1,bfloat16,bfloat16,0,1.9455472946166992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,1,128,1,fp8,fp8,0,1.613088035583496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,2,128,1,bfloat16,fp8,0,1.5766176223754882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,2,128,1,bfloat16,bfloat16,0,1.734227180480957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,2,128,1,fp8,fp8,0,1.6953216552734376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,4,128,1,bfloat16,bfloat16,0,1.8545183181762694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,4,128,1,bfloat16,fp8,0,1.5590448379516602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,96,128,1,bfloat16,bfloat16,0,1.0882736206054688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,96,128,1,bfloat16,fp8,0,0.9101696014404297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,4,128,1,fp8,fp8,0,1.6447071075439452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,8,128,1,bfloat16,fp8,0,1.5822128295898437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,8,128,1,fp8,fp8,0,1.5825839996337892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,96,128,1,fp8,fp8,0,1.0812031745910644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,96,8,128,1,bfloat16,bfloat16,0,1.9987808227539063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,1,128,1,fp8,fp8,0,0.7881135940551758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,1,128,1,bfloat16,bfloat16,0,0.8274959564208985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,1,128,1,bfloat16,fp8,0,0.8737168312072754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,2,128,1,bfloat16,bfloat16,0,0.837889575958252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,2,128,1,bfloat16,fp8,0,0.7898687839508056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,2,128,1,fp8,fp8,0,0.8042703628540039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,4,128,1,bfloat16,bfloat16,0,0.9303263664245606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,4,128,1,bfloat16,fp8,0,0.9855263710021973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,4,128,1,fp8,fp8,0,0.7933472156524658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,8,128,1,bfloat16,bfloat16,0,0.8567088127136231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,96,128,1,bfloat16,fp8,0,0.4607056140899658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,8,128,1,bfloat16,fp8,0,0.8241024017333984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,96,128,1,bfloat16,bfloat16,0,0.5319503784179688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,96,8,128,1,fp8,fp8,0,0.7975232124328613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,96,128,1,fp8,fp8,0,0.5031824111938477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,1,128,1,bfloat16,bfloat16,0,0.4812640190124512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,2,128,1,bfloat16,bfloat16,0,0.4243840217590332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,1,128,1,fp8,fp8,0,0.4302688121795654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,2,128,1,bfloat16,fp8,0,0.4017343997955322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,2,128,1,fp8,fp8,0,0.4009103775024414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,4,128,1,bfloat16,bfloat16,0,0.42737598419189454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,4,128,1,bfloat16,fp8,0,0.4312128067016602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,4,128,1,fp8,fp8,0,0.4298880100250244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,8,128,1,fp8,fp8,0,0.40428638458251953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,8,128,1,bfloat16,fp8,0,0.4344655990600586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,8,128,1,bfloat16,bfloat16,0,0.4323135852813721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,96,1,128,1,bfloat16,fp8,0,0.40047202110290525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,1,128,1,bfloat16,fp8,0,12.224539184570313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,1,128,1,fp8,fp8,0,12.446988677978515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,2,128,1,fp8,fp8,0,12.255915069580078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,2,128,1,bfloat16,fp8,0,12.335862731933593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,1,128,1,bfloat16,bfloat16,0,14.288743591308593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,2,128,1,bfloat16,bfloat16,0,14.100137329101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,4,128,1,bfloat16,bfloat16,0,14.793336486816406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,4,128,1,bfloat16,fp8,0,12.359442901611327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,96,128,1,bfloat16,fp8,0,7.331378936767578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,96,128,1,fp8,fp8,0,7.36902084350586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,1,128,1,bfloat16,bfloat16,0,7.087628936767578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,96,128,1,bfloat16,bfloat16,0,9.242350769042968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,4,128,1,fp8,fp8,0,12.371018981933593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,8,128,1,bfloat16,fp8,0,12.440009307861327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,8,128,1,fp8,fp8,0,12.962384033203126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,96,8,128,1,bfloat16,bfloat16,0,14.923373413085937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,1,128,1,bfloat16,fp8,0,6.124054336547852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,1,128,1,fp8,fp8,0,6.125070571899414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,2,128,1,bfloat16,bfloat16,0,7.292256164550781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,2,128,1,bfloat16,fp8,0,6.37532958984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,2,128,1,fp8,fp8,0,6.122444915771484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,4,128,1,bfloat16,bfloat16,0,7.522013092041016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,4,128,1,bfloat16,fp8,0,6.299177551269532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,4,128,1,fp8,fp8,0,6.641336059570312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,96,128,1,bfloat16,fp8,0,3.721500778198242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,96,128,1,bfloat16,bfloat16,0,4.693899154663086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,8,128,1,bfloat16,fp8,0,6.218852615356445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,8,128,1,bfloat16,bfloat16,0,7.404670715332031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,96,8,128,1,fp8,fp8,0,6.222707366943359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,96,128,1,fp8,fp8,0,3.8128097534179686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,1,128,1,bfloat16,bfloat16,0,3.67694091796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,1,128,1,bfloat16,fp8,0,3.0887680053710938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,1,128,1,fp8,fp8,0,3.079145622253418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,2,128,1,bfloat16,fp8,0,3.0738048553466797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,2,128,1,bfloat16,bfloat16,0,3.5239295959472656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,2,128,1,fp8,fp8,0,3.177204895019531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,4,128,1,bfloat16,fp8,0,3.097737693786621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,4,128,1,bfloat16,bfloat16,0,3.510396957397461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,4,128,1,fp8,fp8,0,3.23022575378418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,8,128,1,bfloat16,bfloat16,0,3.329502487182617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,96,128,1,bfloat16,fp8,0,1.8454719543457032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,96,128,1,bfloat16,bfloat16,0,2.2091840744018554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,8,128,1,fp8,fp8,0,3.1065439224243163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,96,128,1,fp8,fp8,0,1.8462207794189454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,1,128,1,bfloat16,bfloat16,0,1.7458351135253907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,1,128,1,bfloat16,fp8,0,1.549937629699707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,2,128,1,bfloat16,bfloat16,0,1.643671989440918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,1,128,1,fp8,fp8,0,1.7247615814208985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,2,128,1,bfloat16,fp8,0,1.5868816375732422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,2,128,1,fp8,fp8,0,1.542294406890869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,4,128,1,bfloat16,fp8,0,1.6108959197998047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,4,128,1,fp8,fp8,0,1.663649559020996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,96,128,1,bfloat16,bfloat16,0,1.084449577331543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,8,128,1,bfloat16,fp8,0,1.635628890991211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,8,128,1,bfloat16,bfloat16,0,1.8330671310424804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,96,128,1,bfloat16,fp8,0,0.9322239875793457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,8,128,1,fp8,fp8,0,1.5562687873840333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,96,8,128,1,bfloat16,fp8,0,3.1011104583740234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,96,128,1,fp8,fp8,0,0.9293487548828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,1,128,1,bfloat16,bfloat16,0,0.8143088340759277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,1,128,1,bfloat16,fp8,0,0.8073328018188477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,2,128,1,bfloat16,bfloat16,0,0.8379023551940918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,1,128,1,fp8,fp8,0,0.8840592384338379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,2,128,1,bfloat16,fp8,0,0.774457597732544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,2,128,1,fp8,fp8,0,0.76976318359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,4,128,1,bfloat16,bfloat16,0,0.9401935577392578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,4,128,1,bfloat16,fp8,0,0.8273136138916015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,4,128,1,fp8,fp8,0,0.8036335945129395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,8,128,1,bfloat16,bfloat16,0,0.8522144317626953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,8,128,1,bfloat16,fp8,0,0.8324848175048828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,96,8,128,1,fp8,fp8,0,0.7841375827789306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,96,128,1,bfloat16,bfloat16,0,0.5432544231414795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,1,128,1,bfloat16,bfloat16,0,0.4212800025939941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,96,128,1,fp8,fp8,0,0.5022128105163575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,1,128,1,bfloat16,fp8,0,0.40615358352661135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,1,128,1,fp8,fp8,0,0.40282721519470216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,2,128,1,bfloat16,fp8,0,0.39433441162109373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,96,4,128,1,bfloat16,bfloat16,0,1.7328208923339843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,2,128,1,fp8,fp8,0,0.40264320373535156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,4,128,1,bfloat16,bfloat16,0,0.4373216152191162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,4,128,1,bfloat16,fp8,0,0.41118879318237306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,8,128,1,bfloat16,bfloat16,0,0.42218718528747556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,96,128,1,bfloat16,bfloat16,0,0.2898224115371704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,8,128,1,fp8,fp8,0,0.40660958290100097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,8,128,1,bfloat16,fp8,0,0.404150390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,96,128,1,bfloat16,fp8,0,0.25101280212402344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,96,128,1,fp8,fp8,0,0.24446721076965333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,1,128,1,bfloat16,bfloat16,0,0.22140638828277587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,1,128,1,bfloat16,fp8,0,0.20628640651702881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,1,128,1,fp8,fp8,0,0.20493121147155763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,2,128,1,bfloat16,bfloat16,0,0.22121119499206543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,2,128,1,bfloat16,fp8,0,0.202728009223938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,2,128,1,fp8,fp8,0,0.21033599376678466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,4,128,1,bfloat16,bfloat16,0,0.22047998905181884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,4,128,1,bfloat16,fp8,0,0.2061919927597046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,4,128,1,fp8,fp8,0,0.2082672119140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,8,128,1,bfloat16,bfloat16,0,0.21815040111541747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,8,128,1,bfloat16,fp8,0,0.20974559783935548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,96,8,128,1,fp8,fp8,0,0.20905280113220215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,96,128,1,bfloat16,fp8,0,0.48052639961242677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,2,128,1,bfloat16,bfloat16,0,0.4211616039276123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,96,4,128,1,fp8,fp8,0,0.3967152118682861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,1,128,1,bfloat16,fp8,0,7.6096336364746096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,1,128,1,fp8,fp8,0,7.633022308349609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,2,128,1,bfloat16,fp8,0,7.620800018310547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,1,128,1,bfloat16,bfloat16,0,8.540329742431641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,2,128,1,bfloat16,bfloat16,0,8.552260589599609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,2,128,1,fp8,fp8,0,7.651725006103516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,4,128,1,bfloat16,fp8,0,7.618531036376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,4,128,1,bfloat16,bfloat16,0,8.597994995117187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,96,128,1,bfloat16,fp8,0,4.7385200500488285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,96,128,1,bfloat16,bfloat16,0,5.866611099243164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,1,128,1,bfloat16,bfloat16,0,4.259936141967773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,96,128,1,fp8,fp8,0,4.726988983154297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,4,128,1,fp8,fp8,0,7.631745910644531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,8,128,1,bfloat16,fp8,0,7.652299499511718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,8,128,1,fp8,fp8,0,7.685508728027344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,96,8,128,1,bfloat16,bfloat16,0,8.588362884521484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,1,128,1,bfloat16,fp8,0,4.0424446105957035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,1,128,1,fp8,fp8,0,3.7974624633789062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,2,128,1,bfloat16,fp8,0,3.807555389404297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,2,128,1,fp8,fp8,0,3.801113510131836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,2,128,1,bfloat16,bfloat16,0,4.235758590698242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,4,128,1,bfloat16,fp8,0,3.8104591369628906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,4,128,1,bfloat16,bfloat16,0,4.204391860961914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,96,128,1,bfloat16,bfloat16,0,2.801678466796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,96,128,1,bfloat16,fp8,0,2.363107109069824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,4,128,1,fp8,fp8,0,3.8218830108642576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,96,128,1,fp8,fp8,0,2.5415311813354493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,8,128,1,bfloat16,fp8,0,3.8514942169189452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,8,128,1,bfloat16,bfloat16,0,4.366896057128907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,96,8,128,1,fp8,fp8,0,3.826366424560547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,1,128,1,bfloat16,fp8,0,1.8830047607421876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,1,128,1,fp8,fp8,0,1.9009456634521484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,1,128,1,bfloat16,bfloat16,0,2.089583969116211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,2,128,1,bfloat16,bfloat16,0,2.0943151473999024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,2,128,1,bfloat16,fp8,0,1.9030847549438477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,2,128,1,fp8,fp8,0,1.9139087677001954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,4,128,1,bfloat16,bfloat16,0,2.0933040618896483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,4,128,1,bfloat16,fp8,0,1.8994512557983398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,4,128,1,fp8,fp8,0,1.9175615310668945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,96,128,1,bfloat16,bfloat16,0,1.3890815734863282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,8,128,1,bfloat16,bfloat16,0,2.1192272186279295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,96,128,1,bfloat16,fp8,0,1.3627424240112305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,8,128,1,bfloat16,fp8,0,1.9912223815917969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,96,8,128,1,fp8,fp8,0,1.9312591552734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,1,128,1,bfloat16,bfloat16,0,0.9879055976867676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,1,128,1,bfloat16,fp8,0,1.0285183906555175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,96,128,1,fp8,fp8,0,1.1916080474853517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,1,128,1,fp8,fp8,0,0.9788543701171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,2,128,1,bfloat16,bfloat16,0,1.074227237701416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,2,128,1,fp8,fp8,0,0.9645024299621582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,2,128,1,bfloat16,fp8,0,0.9590208053588867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,4,128,1,bfloat16,bfloat16,0,1.0575856208801269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,4,128,1,bfloat16,fp8,0,1.0009488105773925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,4,128,1,fp8,fp8,0,0.956935977935791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,8,128,1,bfloat16,bfloat16,0,1.0191295623779297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,96,128,1,bfloat16,bfloat16,0,0.7062592029571533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,8,128,1,bfloat16,fp8,0,1.0002079963684083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,96,8,128,1,fp8,fp8,0,0.9667296409606934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,1,128,1,bfloat16,bfloat16,0,0.49615039825439455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,96,128,1,bfloat16,fp8,0,0.6198160171508789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,96,128,1,fp8,fp8,0,0.6187104225158692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,1,128,1,bfloat16,fp8,0,0.4873744010925293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,1,128,1,fp8,fp8,0,0.49288158416748046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,2,128,1,bfloat16,bfloat16,0,0.5053247928619384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,2,128,1,bfloat16,fp8,0,0.4835951805114746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,4,128,1,bfloat16,bfloat16,0,0.5170159816741944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,4,128,1,bfloat16,fp8,0,0.49002561569213865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,4,128,1,fp8,fp8,0,0.4945199966430664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,8,128,1,bfloat16,bfloat16,0,0.5192255973815918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,96,128,1,bfloat16,bfloat16,0,0.3583807945251465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,8,128,1,bfloat16,fp8,0,0.5011280059814454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,1,128,1,bfloat16,bfloat16,0,0.25707039833068845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,96,128,1,fp8,fp8,0,0.31950879096984863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,1,128,1,bfloat16,fp8,0,0.24899039268493653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,1,128,1,fp8,fp8,0,0.24632320404052735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,2,128,1,bfloat16,bfloat16,0,0.2660752058029175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,2,128,1,bfloat16,fp8,0,0.2482815980911255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,2,128,1,fp8,fp8,0,0.25381920337677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,4,128,1,bfloat16,fp8,0,0.24802238941192628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,4,128,1,bfloat16,bfloat16,0,0.2713200092315674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,4,128,1,fp8,fp8,0,0.27174079418182373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,8,128,1,bfloat16,bfloat16,0,0.2642159938812256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,8,128,1,bfloat16,fp8,0,0.2563744068145752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,96,128,1,bfloat16,bfloat16,0,0.19213279485702514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,8,128,1,fp8,fp8,0,0.24980640411376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,96,128,1,bfloat16,fp8,0,0.17677119970321656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,96,128,1,fp8,fp8,0,0.16310399770736694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,1,128,1,bfloat16,bfloat16,0,0.1383952021598816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,1,128,1,bfloat16,fp8,0,0.13374240398406984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,1,128,1,fp8,fp8,0,0.12843199968338012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,2,128,1,bfloat16,bfloat16,0,0.14934560060501098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,2,128,1,fp8,fp8,0,0.1321120023727417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,4,128,1,bfloat16,bfloat16,0,0.14173920154571534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,4,128,1,bfloat16,fp8,0,0.1291856050491333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,4,128,1,fp8,fp8,0,0.13789119720458984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,8,128,1,bfloat16,bfloat16,0,0.14241119623184204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,8,128,1,bfloat16,fp8,0,0.13533920049667358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,8,128,1,fp8,fp8,0,0.13006720542907715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,2,128,1,fp8,fp8,0,0.4887199878692627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,96,8,128,1,fp8,fp8,0,0.4932991981506348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,96,96,128,1,bfloat16,fp8,0,0.31653759479522703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,96,2,128,1,bfloat16,fp8,0,0.12927839756011963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,1,128,1,bfloat16,fp8,0,8.05643539428711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,1,128,1,fp8,fp8,0,8.034552001953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,1,128,1,bfloat16,bfloat16,0,8.875291442871093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,2,128,1,bfloat16,bfloat16,0,8.868315124511719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,2,128,1,bfloat16,fp8,0,7.964424133300781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,2,128,1,fp8,fp8,0,7.972801971435547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,4,128,1,bfloat16,bfloat16,0,8.539323425292968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,4,128,1,bfloat16,fp8,0,8.03025131225586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,96,128,1,bfloat16,fp8,0,5.229044723510742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,4,128,1,fp8,fp8,0,8.042842864990234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,96,128,1,bfloat16,bfloat16,0,6.384283065795898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,8,128,1,bfloat16,fp8,0,8.117137908935547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,8,128,1,fp8,fp8,0,8.095478057861328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,96,128,1,fp8,fp8,0,5.238859176635742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,96,8,128,1,bfloat16,bfloat16,0,9.097571563720702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,1,128,1,bfloat16,bfloat16,0,4.31921615600586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,1,128,1,bfloat16,fp8,0,3.989384078979492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,1,128,1,fp8,fp8,0,3.987696075439453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,2,128,1,fp8,fp8,0,3.9816513061523438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,2,128,1,bfloat16,bfloat16,0,4.171500778198242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,2,128,1,bfloat16,fp8,0,3.9767967224121095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,4,128,1,bfloat16,fp8,0,4.03466567993164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,4,128,1,bfloat16,bfloat16,0,4.240836715698242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,1,128,1,bfloat16,bfloat16,0,2.065804862976074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,4,128,1,fp8,fp8,0,4.054755020141601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,96,128,1,fp8,fp8,0,2.63317928314209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,8,128,1,bfloat16,fp8,0,4.043172836303711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,96,128,1,bfloat16,bfloat16,0,3.280944061279297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,8,128,1,fp8,fp8,0,4.044705581665039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,1,128,1,bfloat16,fp8,0,2.00008487701416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,1,128,1,fp8,fp8,0,1.9931600570678711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,2,128,1,bfloat16,bfloat16,0,2.053553581237793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,2,128,1,bfloat16,fp8,0,1.9957712173461915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,2,128,1,fp8,fp8,0,2.0334896087646483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,4,128,1,bfloat16,bfloat16,0,2.0650304794311523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,96,128,1,bfloat16,fp8,0,2.6156543731689452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,4,128,1,bfloat16,fp8,0,1.999679946899414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,96,8,128,1,bfloat16,bfloat16,0,4.461401748657226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,4,128,1,fp8,fp8,0,2.0081615447998047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,8,128,1,bfloat16,bfloat16,0,2.182646369934082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,8,128,1,bfloat16,fp8,0,2.0270816802978517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,96,8,128,1,fp8,fp8,0,2.0375680923461914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,96,128,1,bfloat16,fp8,0,1.3147040367126466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,96,128,1,bfloat16,bfloat16,0,1.614072036743164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,1,128,1,bfloat16,bfloat16,0,1.022283172607422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,1,128,1,bfloat16,fp8,0,0.9988880157470703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,96,128,1,fp8,fp8,0,1.3127615928649903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,1,128,1,fp8,fp8,0,1.0213775634765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,2,128,1,bfloat16,bfloat16,0,1.0184831619262695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,2,128,1,bfloat16,fp8,0,1.0081376075744628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,2,128,1,fp8,fp8,0,1.0082863807678222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,4,128,1,bfloat16,bfloat16,0,1.0635168075561523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,4,128,1,bfloat16,fp8,0,1.0233072280883788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,4,128,1,fp8,fp8,0,1.009607982635498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,8,128,1,bfloat16,bfloat16,0,1.0486751556396485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,96,128,1,bfloat16,bfloat16,0,0.7934847831726074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,96,128,1,bfloat16,fp8,0,0.6688384056091309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,1,128,1,bfloat16,bfloat16,0,0.5137375831604004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,1,128,1,bfloat16,fp8,0,0.5048704147338867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,96,128,1,fp8,fp8,0,0.6854944229125977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,1,128,1,fp8,fp8,0,0.5583712100982666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,2,128,1,fp8,fp8,0,0.5075776100158691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,2,128,1,bfloat16,fp8,0,0.5091663837432862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,4,128,1,bfloat16,bfloat16,0,0.5203824043273926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,4,128,1,bfloat16,fp8,0,0.5258304119110108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,4,128,1,fp8,fp8,0,0.539411211013794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,96,128,1,bfloat16,bfloat16,0,0.40750880241394044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,8,128,1,bfloat16,bfloat16,0,0.5325456142425538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,8,128,1,bfloat16,fp8,0,0.5173312187194824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,8,128,1,fp8,fp8,0,0.5150591850280761
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,96,128,1,bfloat16,fp8,0,0.35188961029052734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,1,128,1,bfloat16,bfloat16,0,0.2635535955429077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,1,128,1,bfloat16,fp8,0,0.2561327934265137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,1,128,1,fp8,fp8,0,0.2626352071762085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,96,128,1,fp8,fp8,0,0.344868803024292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,2,128,1,bfloat16,bfloat16,0,0.2766783952713013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,2,128,1,bfloat16,fp8,0,0.25939359664916994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,2,128,1,fp8,fp8,0,0.2579823970794678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,4,128,1,bfloat16,fp8,0,0.2657504081726074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,4,128,1,bfloat16,bfloat16,0,0.27148640155792236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,4,128,1,fp8,fp8,0,0.2727200031280518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,8,128,1,bfloat16,bfloat16,0,0.27345120906829834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,8,128,1,bfloat16,fp8,0,0.2615135908126831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,96,8,128,1,fp8,fp8,0,0.2679231882095337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,96,128,1,bfloat16,fp8,0,0.18275359869003296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,1,128,1,bfloat16,bfloat16,0,0.1374079942703247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,96,128,1,fp8,fp8,0,0.17764639854431152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,1,128,1,bfloat16,fp8,0,0.1370751976966858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,1,128,1,fp8,fp8,0,0.13951679468154907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,2,128,1,bfloat16,bfloat16,0,0.13869760036468506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,2,128,1,fp8,fp8,0,0.13774559497833253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,4,128,1,bfloat16,bfloat16,0,0.14512640237808228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,4,128,1,bfloat16,fp8,0,0.13500800132751464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,4,128,1,fp8,fp8,0,0.1389631986618042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,8,128,1,bfloat16,bfloat16,0,0.1476704001426697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,8,128,1,bfloat16,fp8,0,0.13634239435195922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,8,128,1,fp8,fp8,0,0.13993920087814332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,96,128,1,bfloat16,bfloat16,0,0.11330080032348633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,96,128,1,bfloat16,fp8,0,0.094651198387146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,96,128,1,fp8,fp8,0,0.09663839936256409
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,1,128,1,bfloat16,bfloat16,0,0.07883999943733215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,8,128,1,fp8,fp8,0,1.065875244140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,1,128,1,bfloat16,fp8,0,0.07081279754638672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,1,128,1,fp8,fp8,0,0.0729312002658844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,2,128,1,bfloat16,bfloat16,0,0.07786399722099305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,2,128,1,bfloat16,fp8,0,0.07286239862442016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,4,128,1,bfloat16,bfloat16,0,0.07922559976577759
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,4,128,1,bfloat16,fp8,0,0.07447199821472168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,4,128,1,fp8,fp8,0,0.07374399900436401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,8,128,1,bfloat16,bfloat16,0,0.07964640259742736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,8,128,1,bfloat16,fp8,0,0.0752623975276947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,96,2,128,1,bfloat16,bfloat16,0,0.5838704109191895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,8,128,1,fp8,fp8,0,0.07523199915885925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,96,128,1,bfloat16,bfloat16,0,0.2129568099975586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,96,2,128,1,bfloat16,fp8,0,0.13474559783935547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,96,8,128,1,bfloat16,fp8,0,1.018454360961914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,96,2,128,1,fp8,fp8,0,0.07117760181427002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,1,128,1,bfloat16,bfloat16,0,5.8170928955078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,1,128,1,fp8,fp8,0,5.729750442504883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,1,128,1,bfloat16,fp8,0,5.795840072631836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,2,128,1,bfloat16,bfloat16,0,5.8285072326660154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,2,128,1,bfloat16,fp8,0,5.745403289794922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,2,128,1,fp8,fp8,0,5.742647933959961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,4,128,1,bfloat16,bfloat16,0,5.95710563659668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,4,128,1,bfloat16,fp8,0,5.7609809875488285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,4,128,1,fp8,fp8,0,5.729587173461914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,8,128,1,bfloat16,bfloat16,0,6.100860977172852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,96,128,1,bfloat16,fp8,0,4.241817474365234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,8,128,1,bfloat16,fp8,0,5.841299057006836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,96,128,1,fp8,fp8,0,4.195406341552735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,96,128,1,bfloat16,bfloat16,0,5.004795074462891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,96,8,128,1,fp8,fp8,0,5.861257553100586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,1,128,1,bfloat16,bfloat16,0,2.8844703674316405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,1,128,1,bfloat16,fp8,0,2.832769584655762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,1,128,1,fp8,fp8,0,2.853455924987793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,2,128,1,bfloat16,bfloat16,0,2.876851272583008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,2,128,1,bfloat16,fp8,0,2.8419504165649414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,2,128,1,fp8,fp8,0,2.8426719665527345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,4,128,1,bfloat16,bfloat16,0,2.955936050415039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,4,128,1,bfloat16,fp8,0,2.8883455276489256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,4,128,1,fp8,fp8,0,2.8974767684936524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,8,128,1,bfloat16,bfloat16,0,3.016169548034668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,8,128,1,bfloat16,fp8,0,2.9359216690063477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,1,128,1,bfloat16,bfloat16,0,1.4294480323791503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,96,128,1,bfloat16,fp8,0,2.10164794921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,1,128,1,bfloat16,fp8,0,1.4292688369750977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,96,128,1,fp8,fp8,0,2.14880485534668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,96,128,1,bfloat16,bfloat16,0,2.5239999771118162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,1,128,1,fp8,fp8,0,1.4335247993469238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,96,8,128,1,fp8,fp8,0,2.9388608932495117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,2,128,1,bfloat16,bfloat16,0,1.4407999992370606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,2,128,1,bfloat16,fp8,0,1.4421504020690918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,2,128,1,fp8,fp8,0,1.4278512001037598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,4,128,1,bfloat16,bfloat16,0,1.4643808364868165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,4,128,1,bfloat16,fp8,0,1.461689567565918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,4,128,1,fp8,fp8,0,1.4665167808532715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,8,128,1,bfloat16,bfloat16,0,1.5108991622924806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,8,128,1,bfloat16,fp8,0,1.4745648384094239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,96,8,128,1,fp8,fp8,0,1.4895423889160155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,96,128,1,bfloat16,bfloat16,0,1.2670080184936523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,1,128,1,bfloat16,bfloat16,0,0.7236048221588135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,96,128,1,bfloat16,fp8,0,1.0578351974487306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,1,128,1,bfloat16,fp8,0,0.7294943809509278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,1,128,1,fp8,fp8,0,0.7241536140441894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,2,128,1,bfloat16,bfloat16,0,0.7260960102081299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,96,128,1,fp8,fp8,0,1.0668975830078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,2,128,1,bfloat16,fp8,0,0.7261727809906006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,2,128,1,fp8,fp8,0,0.7242159843444824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,4,128,1,bfloat16,bfloat16,0,0.741651201248169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,4,128,1,bfloat16,fp8,0,0.7288976192474366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,4,128,1,fp8,fp8,0,0.7351647853851319
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,8,128,1,bfloat16,bfloat16,0,0.7637839794158936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,8,128,1,bfloat16,fp8,0,0.7442351818084717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,96,128,1,bfloat16,bfloat16,0,0.6460095882415772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,96,8,128,1,fp8,fp8,0,0.7459839820861817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,96,128,1,bfloat16,fp8,0,0.538321590423584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,1,128,1,bfloat16,bfloat16,0,0.3654191970825195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,1,128,1,bfloat16,fp8,0,0.36876640319824217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,96,128,1,fp8,fp8,0,0.5357647895812988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,1,128,1,fp8,fp8,0,0.3681999921798706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,2,128,1,bfloat16,bfloat16,0,0.3691168069839478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,2,128,1,fp8,fp8,0,0.3676048040390015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,4,128,1,bfloat16,bfloat16,0,0.3799391984939575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,4,128,1,bfloat16,fp8,0,0.36922879219055177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,4,128,1,fp8,fp8,0,0.37325119972229004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,8,128,1,bfloat16,bfloat16,0,0.3853424072265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,8,128,1,bfloat16,fp8,0,0.37853920459747314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,96,128,1,bfloat16,bfloat16,0,0.3298127889633179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,8,128,1,fp8,fp8,0,0.3784336090087891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,96,128,1,bfloat16,fp8,0,0.27932960987091066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,1,128,1,bfloat16,bfloat16,0,0.1909824013710022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,96,128,1,fp8,fp8,0,0.27534239292144774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,1,128,1,bfloat16,fp8,0,0.18780640363693238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,1,128,1,fp8,fp8,0,0.18810080289840697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,2,128,1,bfloat16,bfloat16,0,0.19602880477905274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,2,128,1,bfloat16,fp8,0,0.19014400243759155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,2,128,1,fp8,fp8,0,0.1936463952064514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,4,128,1,bfloat16,fp8,0,0.19040800333023072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,4,128,1,fp8,fp8,0,0.19029279947280883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,8,128,1,bfloat16,bfloat16,0,0.19962559938430785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,8,128,1,bfloat16,fp8,0,0.19906239509582518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,8,128,1,fp8,fp8,0,0.19471999406814575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,96,128,1,bfloat16,bfloat16,0,0.17518559694290162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,96,128,1,bfloat16,fp8,0,0.14502559900283812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,96,128,1,fp8,fp8,0,0.1440943956375122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,1,128,1,bfloat16,bfloat16,0,0.1009071946144104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,1,128,1,bfloat16,fp8,0,0.10312800407409668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,2,128,1,bfloat16,fp8,0,0.1015056014060974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,2,128,1,fp8,fp8,0,0.10116480588912964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,4,128,1,bfloat16,fp8,0,0.10212160348892212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,4,128,1,fp8,fp8,0,0.10174880027770997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,8,128,1,bfloat16,bfloat16,0,0.10464320182800294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,8,128,1,bfloat16,fp8,0,0.10324000120162964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,8,128,1,fp8,fp8,0,0.103547203540802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,96,128,1,bfloat16,bfloat16,0,0.09219520092010498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,96,128,1,bfloat16,fp8,0,0.08000479936599732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,1,128,1,bfloat16,bfloat16,0,0.058001601696014406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,1,128,1,bfloat16,fp8,0,0.05542399883270264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,1,128,1,fp8,fp8,0,0.055332797765731814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,2,128,1,bfloat16,bfloat16,0,0.058182400465011594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,2,128,1,bfloat16,fp8,0,0.056201601028442384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,2,128,1,fp8,fp8,0,0.05594239830970764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,4,128,1,bfloat16,bfloat16,0,0.05942080020904541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,4,128,1,bfloat16,fp8,0,0.05650240182876587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,4,128,1,fp8,fp8,0,0.056380802392959596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,96,2,128,1,bfloat16,fp8,0,0.37461280822753906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,8,128,1,bfloat16,bfloat16,0,0.06037120223045349
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,8,128,1,bfloat16,fp8,0,0.058057600259780885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,8,128,1,fp8,fp8,0,0.05843520164489746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,96,128,1,bfloat16,bfloat16,0,0.05514879822731018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,96,128,1,bfloat16,fp8,0,0.046239998936653134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,96,128,1,fp8,fp8,0,0.04596959948539734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,1,128,1,bfloat16,fp8,0,0.03112640082836151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,1,128,1,fp8,fp8,0,0.030921599268913268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,2,128,1,bfloat16,bfloat16,0,0.030564799904823303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,2,128,1,bfloat16,fp8,0,0.03107680082321167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,96,4,128,1,bfloat16,bfloat16,0,0.19451680183410644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,2,128,1,fp8,fp8,0,0.031052801012992858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,4,128,1,bfloat16,bfloat16,0,0.03033120036125183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,4,128,1,bfloat16,fp8,0,0.03135200142860413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,4,128,1,fp8,fp8,0,0.030766400694847106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,8,128,1,bfloat16,bfloat16,0,0.03118720054626465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,1,128,1,fp8,fp8,0,0.10124640464782715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,8,128,1,bfloat16,fp8,0,0.03084160089492798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,8,128,1,fp8,fp8,0,0.03127360045909881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,2,128,1,bfloat16,bfloat16,0,0.10370559692382812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,96,4,128,1,bfloat16,bfloat16,0,0.10223200321197509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,96,96,128,1,fp8,fp8,0,0.07827360033988953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,1,128,1,bfloat16,bfloat16,0,2.2310352325439453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,1,128,1,bfloat16,fp8,0,2.3423200607299806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,2,128,1,bfloat16,bfloat16,0,2.2632688522338866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,1,128,1,fp8,fp8,0,2.348676872253418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,96,1,128,1,bfloat16,bfloat16,0,0.030608001351356506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,2,128,1,bfloat16,fp8,0,2.3519311904907227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,2,128,1,fp8,fp8,0,2.3686784744262694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,4,128,1,bfloat16,bfloat16,0,2.3277471542358397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,4,128,1,bfloat16,fp8,0,2.3679967880249024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,4,128,1,fp8,fp8,0,2.377276802062988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,8,128,1,bfloat16,bfloat16,0,2.4137359619140626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,8,128,1,bfloat16,fp8,0,2.419500732421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,96,128,1,bfloat16,bfloat16,0,2.310660743713379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,96,8,128,1,fp8,fp8,0,2.419972801208496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,96,128,1,bfloat16,fp8,0,1.7992719650268554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,1,128,1,bfloat16,bfloat16,0,1.1197999954223632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,1,128,1,bfloat16,fp8,0,1.1770480155944825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,1,128,1,fp8,fp8,0,1.1772671699523927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,2,128,1,bfloat16,bfloat16,0,1.1318384170532227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,4,128,1,bfloat16,bfloat16,0,1.1568464279174804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,2,128,1,fp8,fp8,0,1.1826064109802246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,2,128,1,bfloat16,fp8,0,1.1722432136535645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,96,128,1,fp8,fp8,0,1.7931808471679687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,4,128,1,bfloat16,fp8,0,1.1906751632690429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,4,128,1,fp8,fp8,0,1.193990421295166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,96,128,1,fp8,fp8,0,0.9042736053466797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,96,128,1,bfloat16,fp8,0,0.8980192184448242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,8,128,1,bfloat16,bfloat16,0,1.2114175796508788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,8,128,1,fp8,fp8,0,1.2116975784301758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,96,128,1,bfloat16,bfloat16,0,1.1666159629821777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,1,128,1,bfloat16,bfloat16,0,0.5661039829254151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,1,128,1,bfloat16,fp8,0,0.5886832237243652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,2,128,1,bfloat16,bfloat16,0,0.57085280418396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,2,128,1,bfloat16,fp8,0,0.5952447891235352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,1,128,1,fp8,fp8,0,0.595143985748291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,2,128,1,fp8,fp8,0,0.5928895950317383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,4,128,1,bfloat16,bfloat16,0,0.5853295803070069
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,4,128,1,bfloat16,fp8,0,0.6018767833709717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,4,128,1,fp8,fp8,0,0.5987135887145996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,8,128,1,bfloat16,bfloat16,0,0.6094736099243164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,8,128,1,bfloat16,fp8,0,0.6132031917572022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,96,8,128,1,fp8,fp8,0,0.615718412399292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,96,128,1,bfloat16,fp8,0,0.45827198028564453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,96,128,1,bfloat16,bfloat16,0,0.5901584148406982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,96,128,1,fp8,fp8,0,0.45865440368652344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,1,128,1,bfloat16,bfloat16,0,0.28803040981292727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,1,128,1,bfloat16,fp8,0,0.300932788848877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,1,128,1,fp8,fp8,0,0.30212481021881105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,2,128,1,bfloat16,bfloat16,0,0.2918064117431641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,2,128,1,bfloat16,fp8,0,0.303548789024353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,2,128,1,fp8,fp8,0,0.3017647981643677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,4,128,1,bfloat16,bfloat16,0,0.3003743886947632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,4,128,1,bfloat16,fp8,0,0.3061728000640869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,4,128,1,fp8,fp8,0,0.3060863971710205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,8,128,1,bfloat16,bfloat16,0,0.3121776103973389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,8,128,1,bfloat16,fp8,0,0.3115839958190918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,96,8,128,1,fp8,fp8,0,0.3127487897872925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,96,128,1,bfloat16,fp8,0,0.23667359352111816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,96,128,1,fp8,fp8,0,0.23633599281311035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,96,128,1,bfloat16,bfloat16,0,0.30295519828796386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,1,128,1,bfloat16,fp8,0,0.15714399814605712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,1,128,1,fp8,fp8,0,0.1561568021774292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,2,128,1,bfloat16,bfloat16,0,0.15196479558944703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,2,128,1,bfloat16,fp8,0,0.158241605758667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,4,128,1,bfloat16,bfloat16,0,0.15618879795074464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,2,128,1,fp8,fp8,0,0.15688480138778688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,4,128,1,bfloat16,fp8,0,0.15886399745941163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,4,128,1,fp8,fp8,0,0.1598688006401062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,8,128,1,bfloat16,bfloat16,0,0.16268479824066162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,8,128,1,bfloat16,fp8,0,0.16230399608612062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,8,128,1,fp8,fp8,0,0.16159520149230958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,96,128,1,bfloat16,bfloat16,0,0.16022720336914062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,96,128,1,fp8,fp8,0,0.1262992024421692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,1,128,1,bfloat16,bfloat16,0,0.0812175989151001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,1,128,1,bfloat16,fp8,0,0.08433120250701905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,1,128,1,fp8,fp8,0,0.08425279855728149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,2,128,1,bfloat16,bfloat16,0,0.08220000267028808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,2,128,1,bfloat16,fp8,0,0.08536480069160461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,96,8,128,1,bfloat16,fp8,0,1.2134176254272462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,2,128,1,fp8,fp8,0,0.08459839820861817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,4,128,1,bfloat16,bfloat16,0,0.08399999737739564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,4,128,1,bfloat16,fp8,0,0.08631839752197265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,4,128,1,fp8,fp8,0,0.08599039912223816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,8,128,1,bfloat16,fp8,0,0.08693600296974183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,8,128,1,bfloat16,bfloat16,0,0.08669440150260925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,96,128,1,bfloat16,bfloat16,0,0.08662559986114501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,96,128,1,bfloat16,fp8,0,0.06908640265464783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,96,128,1,fp8,fp8,0,0.0687391996383667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,1,128,1,bfloat16,bfloat16,0,0.04782240092754364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,1,128,1,bfloat16,fp8,0,0.047440001368522645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,1,128,1,fp8,fp8,0,0.04707199931144714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,2,128,1,bfloat16,bfloat16,0,0.0470880001783371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,2,128,1,bfloat16,fp8,0,0.04780319929122925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,2,128,1,fp8,fp8,0,0.047886401414871216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,4,128,1,bfloat16,bfloat16,0,0.04821920096874237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,4,128,1,bfloat16,fp8,0,0.047723200917243955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,4,128,1,fp8,fp8,0,0.048982399702072146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,8,128,1,bfloat16,bfloat16,0,0.04986239969730377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,8,128,1,bfloat16,fp8,0,0.04956479966640472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,96,8,128,1,fp8,fp8,0,0.04896480143070221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,96,128,1,bfloat16,bfloat16,0,0.050684797763824466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,96,128,1,bfloat16,fp8,0,0.04057280123233795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,1,128,1,bfloat16,bfloat16,0,0.025190401077270507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,1,128,1,bfloat16,fp8,0,0.027639999985694885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,1,128,1,fp8,fp8,0,0.026817598938941957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,2,128,1,bfloat16,bfloat16,0,0.025102400779724122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,2,128,1,bfloat16,fp8,0,0.02735520005226135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,2,128,1,fp8,fp8,0,0.026800000667572023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,4,128,1,bfloat16,fp8,0,0.027118399739265442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,4,128,1,fp8,fp8,0,0.027046400308609008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,8,128,1,bfloat16,bfloat16,0,0.02629759907722473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,8,128,1,bfloat16,fp8,0,0.027065598964691163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,8,128,1,fp8,fp8,0,0.02736319899559021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,96,128,1,bfloat16,bfloat16,0,0.02927680015563965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,96,128,1,bfloat16,fp8,0,0.022734400629997254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,96,128,1,fp8,fp8,0,0.022332799434661866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,1,128,1,bfloat16,bfloat16,0,0.019012799859046935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,1,128,1,bfloat16,fp8,0,0.018320000171661376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,1,128,1,fp8,fp8,0,0.018294399976730345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,2,128,1,bfloat16,bfloat16,0,0.018848000466823576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,2,128,1,bfloat16,fp8,0,0.018503999710083006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,2,128,1,fp8,fp8,0,0.01845120042562485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,4,128,1,bfloat16,bfloat16,0,0.01913439929485321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,4,128,1,bfloat16,fp8,0,0.018300800025463103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,4,128,1,fp8,fp8,0,0.018571199476718904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,8,128,1,bfloat16,bfloat16,0,0.01892320066690445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,8,128,1,bfloat16,fp8,0,0.018595199286937713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,96,8,128,1,fp8,fp8,0,0.018254399299621582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,96,1,128,1,bfloat16,bfloat16,0,0.1508128046989441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,96,128,1,bfloat16,fp8,0,0.1259328007698059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,1,128,1,bfloat16,bfloat16,0,1.0521984100341797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,1,128,1,bfloat16,fp8,0,1.1340944290161132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,96,8,128,1,fp8,fp8,0,0.0869920015335083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,1,128,1,fp8,fp8,0,1.1365856170654296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,96,128,1,fp8,fp8,0,0.040424001216888425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,2,128,1,bfloat16,fp8,0,1.1487088203430176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,2,128,1,fp8,fp8,0,1.141312026977539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,4,128,1,bfloat16,bfloat16,0,1.0780655860900878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,4,128,1,bfloat16,fp8,0,1.1639103889465332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,4,128,1,fp8,fp8,0,1.163167953491211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,8,128,1,bfloat16,bfloat16,0,1.1162367820739747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,8,128,1,bfloat16,fp8,0,1.2003680229187013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,8,128,1,fp8,fp8,0,1.2080623626708984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,96,128,1,bfloat16,bfloat16,0,1.1281200408935548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,96,128,1,bfloat16,fp8,0,0.941487979888916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,1,128,1,bfloat16,bfloat16,0,0.5319503784179688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,96,128,1,fp8,fp8,0,0.9397423744201661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,1,128,1,bfloat16,fp8,0,0.5729551792144776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,1,128,1,fp8,fp8,0,0.5719552040100098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,2,128,1,bfloat16,bfloat16,0,0.5355343818664551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,2,128,1,bfloat16,fp8,0,0.5756624221801758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,2,128,1,fp8,fp8,0,0.5790207862854004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,4,128,1,bfloat16,bfloat16,0,0.5448736190795899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,96,2,128,1,bfloat16,bfloat16,0,1.0580127716064454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,4,128,1,bfloat16,fp8,0,0.5874752044677735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,4,128,1,fp8,fp8,0,0.5866559982299805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,96,4,128,1,bfloat16,bfloat16,0,0.025360000133514405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,8,128,1,bfloat16,bfloat16,0,0.5648223876953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,8,128,1,bfloat16,fp8,0,0.6088592052459717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,96,8,128,1,fp8,fp8,0,0.6112815856933593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,96,128,1,bfloat16,bfloat16,0,0.5765903949737549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,1,128,1,bfloat16,bfloat16,0,0.2735919952392578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,96,128,1,bfloat16,fp8,0,0.47754721641540526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,1,128,1,bfloat16,fp8,0,0.29097280502319334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,1,128,1,fp8,fp8,0,0.290067195892334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,96,128,1,fp8,fp8,0,0.47616162300109866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,2,128,1,bfloat16,bfloat16,0,0.27573280334472655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,2,128,1,fp8,fp8,0,0.2943264007568359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,2,128,1,bfloat16,fp8,0,0.29244959354400635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,4,128,1,bfloat16,bfloat16,0,0.27951200008392335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,4,128,1,fp8,fp8,0,0.2970128059387207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,4,128,1,bfloat16,fp8,0,0.2987584114074707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,8,128,1,bfloat16,bfloat16,0,0.2856719970703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,8,128,1,bfloat16,fp8,0,0.3128607988357544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,96,8,128,1,fp8,fp8,0,0.31241281032562257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,1,128,1,bfloat16,bfloat16,0,0.14224159717559814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,1,128,1,bfloat16,fp8,0,0.14995839595794677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,96,128,1,bfloat16,fp8,0,0.24464800357818603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,96,128,1,bfloat16,bfloat16,0,0.2945231914520264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,96,128,1,fp8,fp8,0,0.24510400295257567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,1,128,1,fp8,fp8,0,0.15143200159072875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,2,128,1,bfloat16,bfloat16,0,0.14228320121765137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,2,128,1,bfloat16,fp8,0,0.1518272042274475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,2,128,1,fp8,fp8,0,0.1514431953430176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,4,128,1,bfloat16,bfloat16,0,0.14528160095214843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,4,128,1,fp8,fp8,0,0.15310399532318114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,4,128,1,bfloat16,fp8,0,0.15159519910812377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,8,128,1,bfloat16,bfloat16,0,0.15005439519882202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,8,128,1,bfloat16,fp8,0,0.1607983946800232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,96,8,128,1,fp8,fp8,0,0.15942399501800536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,96,128,1,bfloat16,fp8,0,0.12789119482040406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,1,128,1,bfloat16,bfloat16,0,0.0765824019908905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,1,128,1,bfloat16,fp8,0,0.0818015992641449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,1,128,1,fp8,fp8,0,0.08145760297775269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,2,128,1,bfloat16,fp8,0,0.08225280046463013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,2,128,1,fp8,fp8,0,0.08192319869995117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,4,128,1,bfloat16,fp8,0,0.08290079832077027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,4,128,1,fp8,fp8,0,0.08337119817733765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,8,128,1,bfloat16,bfloat16,0,0.07980639934539795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,8,128,1,bfloat16,fp8,0,0.08523679971694946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,8,128,1,fp8,fp8,0,0.08466240167617797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,96,128,1,bfloat16,fp8,0,0.06721600294113159
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,96,128,1,bfloat16,bfloat16,0,0.08578879833221435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,96,128,1,fp8,fp8,0,0.06738560199737549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,1,128,1,bfloat16,bfloat16,0,0.04428800046443939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,1,128,1,bfloat16,fp8,0,0.04293439984321594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,1,128,1,fp8,fp8,0,0.04339999854564667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,2,128,1,bfloat16,bfloat16,0,0.044433599710464476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,2,128,1,bfloat16,fp8,0,0.04382559955120087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,2,128,1,fp8,fp8,0,0.04395680129528046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,4,128,1,bfloat16,bfloat16,0,0.04483200013637543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,4,128,1,bfloat16,fp8,0,0.04463520050048828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,8,128,1,bfloat16,bfloat16,0,0.04629440009593964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,8,128,1,bfloat16,fp8,0,0.045623999834060666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,8,128,1,fp8,fp8,0,0.046433600783348086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,96,128,1,bfloat16,bfloat16,0,0.04810880124568939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,96,128,1,fp8,fp8,0,0.03613600134849548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,1,128,1,bfloat16,bfloat16,0,0.022222399711608887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,1,128,1,bfloat16,fp8,0,0.02359039932489395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,1,128,1,fp8,fp8,0,0.02370080053806305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,2,128,1,bfloat16,bfloat16,0,0.022494399547576906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,2,128,1,bfloat16,fp8,0,0.023768000304698944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,2,128,1,fp8,fp8,0,0.0237184002995491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,4,128,1,bfloat16,bfloat16,0,0.022672000527381896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,4,128,1,bfloat16,fp8,0,0.023790399730205535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,4,128,1,fp8,fp8,0,0.023788799345493317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,8,128,1,bfloat16,bfloat16,0,0.023324799537658692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,8,128,1,bfloat16,fp8,0,0.023963199555873872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,8,128,1,fp8,fp8,0,0.02393440008163452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,96,128,1,bfloat16,bfloat16,0,0.027393600344657897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,96,128,1,bfloat16,fp8,0,0.020284800231456755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,96,128,1,fp8,fp8,0,0.020337599515914916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,1,128,1,bfloat16,bfloat16,0,0.017180800437927246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,1,128,1,bfloat16,fp8,0,0.018195199966430663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,96,128,1,bfloat16,bfloat16,0,0.1586143970489502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,96,128,1,fp8,fp8,0,0.12740639448165894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,2,128,1,bfloat16,bfloat16,0,0.01738400012254715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,2,128,1,bfloat16,fp8,0,0.018401600420475006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,4,128,1,bfloat16,bfloat16,0,0.07762560248374939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,2,128,1,fp8,fp8,0,0.018294399976730345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,4,128,1,bfloat16,bfloat16,0,0.017684799432754517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,4,128,1,bfloat16,fp8,0,0.01814880073070526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,4,128,1,fp8,fp8,0,0.01826079934835434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,8,128,1,bfloat16,bfloat16,0,0.017712000012397765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,8,128,1,bfloat16,fp8,0,0.018329599499702455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,8,128,1,fp8,fp8,0,0.018308800458908082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,96,128,1,bfloat16,bfloat16,0,0.01565759927034378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,96,128,1,bfloat16,fp8,0,0.014236800372600555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,96,128,1,fp8,fp8,0,0.01449120044708252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,1,128,1,bfloat16,bfloat16,0,0.012708799540996551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,1,128,1,bfloat16,fp8,0,0.013366399705410004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,1,128,1,fp8,fp8,0,0.0133200004696846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,2,128,1,bfloat16,bfloat16,0,0.012828800082206725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,2,128,1,bfloat16,fp8,0,0.013356800377368926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,2,128,1,fp8,fp8,0,0.013492800295352936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,4,128,1,bfloat16,bfloat16,0,0.01292639970779419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,4,128,1,bfloat16,fp8,0,0.013523200154304504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,4,128,1,fp8,fp8,0,0.013262400031089782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,8,128,1,bfloat16,bfloat16,0,0.013070400059223174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,8,128,1,bfloat16,fp8,0,0.013007999956607818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,96,8,128,1,fp8,fp8,0,0.013519999384880067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,96,4,128,1,fp8,fp8,0,0.044710400700569156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,96,96,128,1,bfloat16,fp8,0,0.03607679903507233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,1,128,1,bfloat16,bfloat16,0,0.5332320213317872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,1,128,1,bfloat16,fp8,0,0.5810224056243897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,1,128,1,fp8,fp8,0,0.575875186920166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,2,128,1,bfloat16,bfloat16,0,0.5343760013580322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,2,128,1,fp8,fp8,0,0.5877488136291504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,2,128,1,bfloat16,fp8,0,0.5823472023010254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,4,128,1,bfloat16,bfloat16,0,0.5484127998352051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,4,128,1,bfloat16,fp8,0,0.5905871868133545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,4,128,1,fp8,fp8,0,0.5945184230804443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,8,128,1,bfloat16,bfloat16,0,0.5657936096191406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,8,128,1,bfloat16,fp8,0,0.6134543895721436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,96,8,128,1,fp8,fp8,0,0.6137872219085694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,96,128,1,bfloat16,bfloat16,0,0.7145455837249756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,1,128,1,bfloat16,bfloat16,0,0.2731760025024414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,96,128,1,bfloat16,fp8,0,0.6827343940734864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,96,1,128,1,fp8,fp8,0,0.01815039962530136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,1,128,1,bfloat16,fp8,0,0.29609920978546145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,96,2,128,1,bfloat16,bfloat16,0,0.07786719799041748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,96,128,1,fp8,fp8,0,0.6831056118011475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,1,128,1,fp8,fp8,0,0.2941519975662231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,2,128,1,bfloat16,bfloat16,0,0.27413599491119384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,2,128,1,bfloat16,fp8,0,0.2988192081451416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,2,128,1,fp8,fp8,0,0.29681921005249023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,4,128,1,bfloat16,fp8,0,0.3018928050994873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,4,128,1,fp8,fp8,0,0.3035088062286377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,8,128,1,bfloat16,bfloat16,0,0.2877887964248657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,8,128,1,bfloat16,fp8,0,0.31296639442443847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,8,128,1,fp8,fp8,0,0.3134335994720459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,1,128,1,bfloat16,bfloat16,0,0.1417631983757019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,96,128,1,bfloat16,bfloat16,0,0.36664319038391113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,96,128,1,fp8,fp8,0,0.3471904039382935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,96,128,1,bfloat16,fp8,0,0.3491663932800293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,1,128,1,fp8,fp8,0,0.15544960498809815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,2,128,1,bfloat16,bfloat16,0,0.14312319755554198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,2,128,1,bfloat16,fp8,0,0.15586400032043457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,2,128,1,fp8,fp8,0,0.15664160251617432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,4,128,1,bfloat16,bfloat16,0,0.14672960042953492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,4,128,1,bfloat16,fp8,0,0.15856640338897704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,4,128,1,fp8,fp8,0,0.15493600368499755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,8,128,1,bfloat16,bfloat16,0,0.1508288025856018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,8,128,1,bfloat16,fp8,0,0.1624959945678711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,8,128,1,fp8,fp8,0,0.1615216016769409
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,96,128,1,bfloat16,bfloat16,0,0.19149279594421387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,96,128,1,bfloat16,fp8,0,0.1810639977455139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,1,128,1,bfloat16,bfloat16,0,0.07650399804115296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,96,128,1,fp8,fp8,0,0.18192319869995116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,1,128,1,bfloat16,fp8,0,0.08316640257835388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,1,128,1,fp8,fp8,0,0.08284159898757934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,2,128,1,bfloat16,bfloat16,0,0.07763360142707824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,2,128,1,bfloat16,fp8,0,0.08256480097770691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,2,128,1,fp8,fp8,0,0.08375679850578308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,4,128,1,bfloat16,bfloat16,0,0.07896000146865845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,4,128,1,bfloat16,fp8,0,0.08348640203475952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,4,128,1,fp8,fp8,0,0.0842736005783081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,8,128,1,bfloat16,bfloat16,0,0.08053280115127563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,8,128,1,bfloat16,fp8,0,0.08575199842453003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,96,8,128,1,fp8,fp8,0,0.08591840267181397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,96,128,1,bfloat16,bfloat16,0,0.1003424048423767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,96,128,1,bfloat16,fp8,0,0.09338560104370117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,96,128,1,fp8,fp8,0,0.09295200109481812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,1,128,1,bfloat16,bfloat16,0,0.04514400064945221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,1,128,1,bfloat16,fp8,0,0.04344159960746765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,1,128,1,fp8,fp8,0,0.04304159879684448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,2,128,1,bfloat16,bfloat16,0,0.04404639899730682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,2,128,1,bfloat16,fp8,0,0.04264959990978241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,2,128,1,fp8,fp8,0,0.04386720061302185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,4,128,1,bfloat16,bfloat16,0,0.045132800936698914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,4,128,1,fp8,fp8,0,0.04515520036220551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,8,128,1,bfloat16,bfloat16,0,0.04611999988555908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,8,128,1,bfloat16,fp8,0,0.04652479887008667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,8,128,1,fp8,fp8,0,0.0460640013217926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,96,128,1,bfloat16,bfloat16,0,0.058430397510528566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,96,128,1,bfloat16,fp8,0,0.048691201210021975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,96,128,1,fp8,fp8,0,0.04940640032291412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,1,128,1,bfloat16,bfloat16,0,0.02260800004005432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,1,128,1,bfloat16,fp8,0,0.023892800509929656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,1,128,1,fp8,fp8,0,0.023956799507141115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,2,128,1,bfloat16,bfloat16,0,0.022678400576114654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,2,128,1,bfloat16,fp8,0,0.024241599440574645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,2,128,1,fp8,fp8,0,0.023843200504779817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,4,128,1,bfloat16,bfloat16,0,0.022563199698925018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,4,128,1,bfloat16,fp8,0,0.02399359941482544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,4,128,1,fp8,fp8,0,0.024321599304676055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,8,128,1,bfloat16,bfloat16,0,0.02290399968624115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,8,128,1,bfloat16,fp8,0,0.024417600035667418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,96,8,128,1,fp8,fp8,0,0.02409760057926178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,96,128,1,bfloat16,bfloat16,0,0.031385600566864014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,96,128,1,bfloat16,fp8,0,0.024160000681877136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,1,128,1,bfloat16,bfloat16,0,0.017313599586486816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,1,128,1,bfloat16,fp8,0,0.018460799753665925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,1,128,1,fp8,fp8,0,0.018345600366592406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,2,128,1,bfloat16,bfloat16,0,0.017215999960899352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,2,128,1,bfloat16,fp8,0,0.018369600176811218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,2,128,1,fp8,fp8,0,0.01828639954328537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,4,128,1,bfloat16,bfloat16,0,0.017323200404644013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,4,128,1,bfloat16,fp8,0,0.018449600040912627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,4,128,1,fp8,fp8,0,0.01839199960231781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,8,128,1,bfloat16,bfloat16,0,0.017484800517559053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,8,128,1,bfloat16,fp8,0,0.018457600474357606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,8,128,1,fp8,fp8,0,0.018438400328159334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,96,128,1,bfloat16,bfloat16,0,0.018488000333309173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,96,128,1,bfloat16,fp8,0,0.01855839937925339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,96,128,1,fp8,fp8,0,0.01852799952030182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,1,128,1,bfloat16,bfloat16,0,0.012508800625801087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,1,128,1,bfloat16,fp8,0,0.01358720064163208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,96,4,128,1,bfloat16,bfloat16,0,0.27921600341796876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,1,128,1,fp8,fp8,0,0.013444800674915314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,2,128,1,bfloat16,bfloat16,0,0.012464000284671784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,2,128,1,bfloat16,fp8,0,0.013011200726032257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,4,128,1,bfloat16,bfloat16,0,0.012651200592517852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,4,128,1,bfloat16,fp8,0,0.013227200508117676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,4,128,1,fp8,fp8,0,0.01297599971294403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,8,128,1,bfloat16,bfloat16,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,8,128,1,bfloat16,fp8,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,8,128,1,fp8,fp8,0,0.013366399705410004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,96,128,1,bfloat16,bfloat16,0,0.014343999326229095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,96,128,1,bfloat16,fp8,0,0.013836799561977387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,96,128,1,fp8,fp8,0,0.013915200531482697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,1,128,1,bfloat16,bfloat16,0,0.012417600303888322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,1,128,1,bfloat16,fp8,0,0.013064000010490417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,1,128,1,fp8,fp8,0,0.013038399815559387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,96,1,128,1,bfloat16,fp8,0,0.152400004863739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,2,128,1,bfloat16,bfloat16,0,0.012585599720478059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,2,128,1,bfloat16,fp8,0,0.013043199479579926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,2,128,1,fp8,fp8,0,0.013067199289798737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,4,128,1,bfloat16,fp8,0,0.013259199261665345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,4,128,1,fp8,fp8,0,0.01311360001564026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,8,128,1,bfloat16,fp8,0,0.013227200508117676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,8,128,1,fp8,fp8,0,0.013204799592494964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,1,128,1,bfloat16,bfloat16,0,0.2759919881820679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,1,128,1,bfloat16,fp8,0,0.2992991924285889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,1,128,1,fp8,fp8,0,0.3002527952194214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,2,128,1,bfloat16,bfloat16,0,0.2749023914337158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,96,4,128,1,bfloat16,fp8,0,0.04475040137767792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,2,128,1,bfloat16,fp8,0,0.30000159740447996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,2,128,1,fp8,fp8,0,0.30011520385742185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,4,128,1,bfloat16,bfloat16,0,0.282044792175293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,4,128,1,bfloat16,fp8,0,0.30262720584869385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,4,128,1,fp8,fp8,0,0.30517599582672117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,96,96,128,1,fp8,fp8,0,0.024139200150966645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,8,128,1,bfloat16,bfloat16,0,0.2924432039260864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,8,128,1,bfloat16,fp8,0,0.3129184007644653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,96,8,128,1,fp8,fp8,0,0.31303839683532714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,1,128,1,bfloat16,bfloat16,0,0.14331040382385254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,96,128,1,bfloat16,bfloat16,0,0.5420112133026123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,96,128,1,bfloat16,fp8,0,0.5443039894104004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,1,128,1,bfloat16,fp8,0,0.15544159412384034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,1,128,1,fp8,fp8,0,0.15695840120315552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,96,128,1,fp8,fp8,0,0.5453023910522461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,96,2,128,1,fp8,fp8,0,0.013713599741458892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,2,128,1,bfloat16,bfloat16,0,0.14482400417327881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,2,128,1,bfloat16,fp8,0,0.15681439638137817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,2,128,1,fp8,fp8,0,0.15903359651565552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,4,128,1,bfloat16,bfloat16,0,0.14745759963989258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,4,128,1,bfloat16,fp8,0,0.1562384009361267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,8,128,1,bfloat16,fp8,0,0.16353600025177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,8,128,1,fp8,fp8,0,0.1597424030303955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,4,128,1,bfloat16,bfloat16,0,0.012302400171756744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,96,8,128,1,bfloat16,bfloat16,0,0.012544000148773193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,96,128,1,bfloat16,bfloat16,0,0.27933440208435056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,96,128,1,bfloat16,fp8,0,0.2807984113693237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,1,128,1,bfloat16,bfloat16,0,0.07773439884185791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,1,128,1,bfloat16,fp8,0,0.08452159762382508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,1,128,1,fp8,fp8,0,0.08377599716186523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,96,128,1,fp8,fp8,0,0.2813407897949219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,2,128,1,bfloat16,bfloat16,0,0.0781823992729187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,2,128,1,bfloat16,fp8,0,0.08528000116348267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,2,128,1,fp8,fp8,0,0.085452800989151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,4,128,1,bfloat16,bfloat16,0,0.07899680137634277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,4,128,1,fp8,fp8,0,0.08621600270271301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,8,128,1,bfloat16,bfloat16,0,0.08244479894638061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,8,128,1,bfloat16,fp8,0,0.08748480081558227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,96,128,1,bfloat16,bfloat16,0,0.14386719465255737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,96,128,1,bfloat16,fp8,0,0.1474176049232483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,1,128,1,bfloat16,bfloat16,0,0.04476000070571899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,1,128,1,bfloat16,fp8,0,0.0441103994846344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,1,128,1,fp8,fp8,0,0.043787199258804324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,2,128,1,bfloat16,bfloat16,0,0.044679999351501465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,2,128,1,bfloat16,fp8,0,0.043889600038528445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,2,128,1,fp8,fp8,0,0.04450879991054535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,4,128,1,bfloat16,bfloat16,0,0.04519039988517761
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,4,128,1,bfloat16,fp8,0,0.04529919922351837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,4,128,1,fp8,fp8,0,0.044577598571777344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,8,128,1,bfloat16,bfloat16,0,0.0467056006193161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,8,128,1,bfloat16,fp8,0,0.047276800870895384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,8,128,1,fp8,fp8,0,0.04737760126590729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,96,128,1,bfloat16,bfloat16,0,0.08004480004310607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,96,128,1,bfloat16,fp8,0,0.07767199873924255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,1,128,1,bfloat16,bfloat16,0,0.022815999388694764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,96,128,1,fp8,fp8,0,0.0769648015499115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,1,128,1,bfloat16,fp8,0,0.024289600551128387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,1,128,1,fp8,fp8,0,0.02423200011253357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,2,128,1,bfloat16,bfloat16,0,0.022763200104236603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,2,128,1,bfloat16,fp8,0,0.024073599278926848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,2,128,1,fp8,fp8,0,0.024240000545978545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,4,128,1,bfloat16,fp8,0,0.02414720058441162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,4,128,1,fp8,fp8,0,0.024320000410079957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,8,128,1,bfloat16,bfloat16,0,0.02305919975042343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,8,128,1,bfloat16,fp8,0,0.02438880056142807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,8,128,1,fp8,fp8,0,0.0244159996509552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,96,128,1,bfloat16,bfloat16,0,0.042900800704956055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,96,128,1,bfloat16,fp8,0,0.03824479877948761
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,96,128,1,fp8,fp8,0,0.038334399461746216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,1,128,1,bfloat16,bfloat16,0,0.017319999635219574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,4,128,1,fp8,fp8,0,0.157916796207428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,1,128,1,bfloat16,fp8,0,0.018503999710083006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,96,8,128,1,bfloat16,bfloat16,0,0.15249760150909425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,1,128,1,fp8,fp8,0,0.018743999302387238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,2,128,1,bfloat16,bfloat16,0,0.01727679967880249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,2,128,1,bfloat16,fp8,0,0.01844319999217987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,2,128,1,fp8,fp8,0,0.018588800728321076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,4,128,1,bfloat16,bfloat16,0,0.01740480065345764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,4,128,1,bfloat16,fp8,0,0.018454399704933167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,8,128,1,bfloat16,bfloat16,0,0.01746560037136078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,8,128,1,bfloat16,fp8,0,0.018566399812698364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,8,128,1,fp8,fp8,0,0.01860159933567047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,96,128,1,bfloat16,bfloat16,0,0.02227199971675873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,96,128,1,bfloat16,fp8,0,0.0233487993478775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,96,128,1,fp8,fp8,0,0.022916799783706664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,1,128,1,bfloat16,bfloat16,0,0.012748800218105316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,1,128,1,bfloat16,fp8,0,0.013279999792575835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,1,128,1,fp8,fp8,0,0.013648000359535218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,4,128,1,bfloat16,fp8,0,0.08539040088653564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,2,128,1,bfloat16,bfloat16,0,0.012734399735927581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,2,128,1,bfloat16,fp8,0,0.013556799292564392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,96,8,128,1,fp8,fp8,0,0.0865887999534607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,2,128,1,fp8,fp8,0,0.013233600556850434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,4,128,1,bfloat16,bfloat16,0,0.012647999823093415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,4,128,1,bfloat16,fp8,0,0.013289600610733032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,8,128,1,bfloat16,bfloat16,0,0.012873600423336028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,8,128,1,bfloat16,fp8,0,0.013255999982357025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,8,128,1,fp8,fp8,0,0.013537600636482239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,96,96,128,1,fp8,fp8,0,0.1472543954849243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,96,128,1,bfloat16,fp8,0,0.018004800379276275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,96,128,1,fp8,fp8,0,0.018177600204944612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,1,128,1,bfloat16,bfloat16,0,0.012587200105190276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,1,128,1,bfloat16,fp8,0,0.013193599879741669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,2,128,1,bfloat16,bfloat16,0,0.012579199671745301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,2,128,1,bfloat16,fp8,0,0.013156799972057343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,2,128,1,fp8,fp8,0,0.013350400328636169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,4,128,1,bfloat16,bfloat16,0,0.012401600182056428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,4,128,1,bfloat16,fp8,0,0.013380800187587739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,4,128,1,fp8,fp8,0,0.012995199859142303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,8,128,1,bfloat16,bfloat16,0,0.012839999794960023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,8,128,1,bfloat16,fp8,0,0.013059200346469879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,8,128,1,fp8,fp8,0,0.013387200236320496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,96,128,1,bfloat16,bfloat16,0,0.01427839994430542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,96,128,1,bfloat16,fp8,0,0.014099200069904328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,96,128,1,fp8,fp8,0,0.013753600418567657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,1,128,1,bfloat16,bfloat16,0,0.012547199428081513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,1,128,1,bfloat16,fp8,0,0.012955200672149659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,1,128,1,fp8,fp8,0,0.013299199938774108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,2,128,1,bfloat16,bfloat16,0,0.012267199903726577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,2,128,1,bfloat16,fp8,0,0.013019199669361114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,2,128,1,fp8,fp8,0,0.012814399600028992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,4,128,1,bfloat16,bfloat16,0,0.012372799962759019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,4,128,1,bfloat16,fp8,0,0.01284160017967224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,4,128,1,fp8,fp8,0,0.01308799982070923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,96,4,128,1,bfloat16,bfloat16,0,0.02300799936056137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,8,128,1,bfloat16,bfloat16,0,0.012206400185823441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,8,128,1,bfloat16,fp8,0,0.013153600692749023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,96,8,128,1,fp8,fp8,0,0.013096000254154205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,1,128,1,bfloat16,bfloat16,0,0.14246560335159303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,1,128,1,bfloat16,fp8,0,0.15368959903717042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,1,128,1,fp8,fp8,0,0.15612640380859374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,2,128,1,bfloat16,bfloat16,0,0.14288320541381835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,2,128,1,bfloat16,fp8,0,0.1539023995399475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,2,128,1,fp8,fp8,0,0.1560479998588562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,4,128,1,bfloat16,bfloat16,0,0.14756159782409667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,4,128,1,bfloat16,fp8,0,0.16007360219955444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,4,128,1,fp8,fp8,0,0.15804320573806763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,8,128,1,bfloat16,bfloat16,0,0.18332799673080444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,8,128,1,bfloat16,fp8,0,0.1899135947227478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,96,8,128,1,fp8,fp8,0,0.1894960045814514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,96,4,128,1,fp8,fp8,0,0.018535999953746794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,1,128,1,bfloat16,bfloat16,0,0.07735360264778138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,96,128,1,bfloat16,bfloat16,0,0.45983037948608396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,1,128,1,bfloat16,fp8,0,0.08424320220947265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,96,128,1,bfloat16,fp8,0,0.4996528148651123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,1,128,1,fp8,fp8,0,0.08457599878311158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,2,128,1,bfloat16,bfloat16,0,0.07839679718017578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,2,128,1,bfloat16,fp8,0,0.08588960170745849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,2,128,1,fp8,fp8,0,0.08535839915275574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,4,128,1,bfloat16,bfloat16,0,0.0800704002380371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,96,4,128,1,fp8,fp8,0,0.013420799374580383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,4,128,1,bfloat16,fp8,0,0.0865440011024475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,96,128,1,bfloat16,bfloat16,0,0.017851200699806214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,4,128,1,fp8,fp8,0,0.0859167993068695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,8,128,1,bfloat16,bfloat16,0,0.09759520292282105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,8,128,1,bfloat16,fp8,0,0.10323359966278076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,96,1,128,1,fp8,fp8,0,0.013073599338531494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,8,128,1,fp8,fp8,0,0.10295039415359497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,1,128,1,bfloat16,bfloat16,0,0.04698559939861298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,1,128,1,bfloat16,fp8,0,0.04457600116729736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,1,128,1,fp8,fp8,0,0.04461280107498169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,2,128,1,bfloat16,bfloat16,0,0.047460800409317015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,96,128,1,fp8,fp8,0,0.2578687906265259
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,2,128,1,bfloat16,fp8,0,0.0447952002286911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,2,128,1,fp8,fp8,0,0.04518080055713654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,4,128,1,bfloat16,bfloat16,0,0.048132801055908205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,4,128,1,bfloat16,fp8,0,0.04597600102424622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,4,128,1,fp8,fp8,0,0.04599680006504059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,8,128,1,bfloat16,bfloat16,0,0.056524801254272464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,8,128,1,bfloat16,fp8,0,0.05547999739646912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,8,128,1,fp8,fp8,0,0.054971200227737424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,96,128,1,bfloat16,bfloat16,0,0.1256335973739624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,96,128,1,bfloat16,fp8,0,0.13463519811630248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,1,128,1,bfloat16,bfloat16,0,0.023054400086402894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,96,128,1,fp8,fp8,0,0.13312480449676514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,1,128,1,fp8,fp8,0,0.024616000056266785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,2,128,1,bfloat16,bfloat16,0,0.02309119999408722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,2,128,1,bfloat16,fp8,0,0.024564799666404725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,2,128,1,fp8,fp8,0,0.02441119998693466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,4,128,1,bfloat16,bfloat16,0,0.023180800676345825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,4,128,1,bfloat16,fp8,0,0.024412800371646882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,4,128,1,fp8,fp8,0,0.02468000054359436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,8,128,1,bfloat16,bfloat16,0,0.026635199785232544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,8,128,1,bfloat16,fp8,0,0.029076799750328064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,8,128,1,fp8,fp8,0,0.028732800483703615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,96,128,1,bfloat16,bfloat16,0,0.06579840183258057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,96,128,1,bfloat16,fp8,0,0.06458719968795776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,96,128,1,fp8,fp8,0,0.06561279892921448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,1,128,1,bfloat16,bfloat16,0,0.017433600127696992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,1,128,1,bfloat16,fp8,0,0.01884319931268692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,1,128,1,fp8,fp8,0,0.018488000333309173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,2,128,1,bfloat16,bfloat16,0,0.017697599530220032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,2,128,1,bfloat16,fp8,0,0.018483200669288637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,2,128,1,fp8,fp8,0,0.01870400011539459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,4,128,1,bfloat16,bfloat16,0,0.017532800137996674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,4,128,1,bfloat16,fp8,0,0.018745599687099455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,4,128,1,fp8,fp8,0,0.018592000007629395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,8,128,1,bfloat16,bfloat16,0,0.017479999363422392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,8,128,1,bfloat16,fp8,0,0.018555200099945067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,96,8,128,1,fp8,fp8,0,0.018681600689888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,96,128,1,bfloat16,bfloat16,0,0.0334991991519928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,96,128,1,bfloat16,fp8,0,0.03711200058460236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,96,128,1,fp8,fp8,0,0.036646398901939395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,1,128,1,bfloat16,bfloat16,0,0.012878400087356568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,1,128,1,bfloat16,fp8,0,0.013542400300502777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,1,128,1,fp8,fp8,0,0.01353919953107834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,2,128,1,bfloat16,bfloat16,0,0.012851199507713318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,2,128,1,bfloat16,fp8,0,0.013806399703025819
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,2,128,1,fp8,fp8,0,0.013596799969673157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,4,128,1,bfloat16,bfloat16,0,0.012963199615478515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,4,128,1,bfloat16,fp8,0,0.013593600690364837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,4,128,1,fp8,fp8,0,0.013518400490283966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,8,128,1,bfloat16,bfloat16,0,0.012889599800109864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,8,128,1,bfloat16,fp8,0,0.013489599525928497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,96,8,128,1,fp8,fp8,0,0.013462400436401368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,96,128,1,bfloat16,bfloat16,0,0.021976000070571898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,96,128,1,bfloat16,fp8,0,0.02276480048894882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,96,128,1,fp8,fp8,0,0.022737599909305573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,1,128,1,bfloat16,bfloat16,0,0.01265760064125061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,1,128,1,fp8,fp8,0,0.013199999928474426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,1,128,1,bfloat16,fp8,0,0.013195200264453888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,2,128,1,bfloat16,bfloat16,0,0.012403199821710587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,2,128,1,bfloat16,fp8,0,0.013140800595283508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,96,128,1,bfloat16,bfloat16,0,0.23556959629058838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,96,96,128,1,bfloat16,fp8,0,0.2574656009674072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,2,128,1,fp8,fp8,0,0.013350400328636169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,4,128,1,bfloat16,bfloat16,0,0.012433599680662155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,4,128,1,bfloat16,fp8,0,0.013308799266815186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,8,128,1,bfloat16,fp8,0,0.013124799728393555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,8,128,1,fp8,fp8,0,0.013172799348831176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,96,128,1,bfloat16,bfloat16,0,0.01796800047159195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,96,128,1,bfloat16,fp8,0,0.01834080070257187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,96,128,1,fp8,fp8,0,0.018136000633239745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,1,128,1,bfloat16,bfloat16,0,0.012603199481964112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,1,128,1,bfloat16,fp8,0,0.013036799430847169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,1,128,1,fp8,fp8,0,0.013281600177288055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,2,128,1,bfloat16,bfloat16,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,96,1,128,1,bfloat16,fp8,0,0.02468319982290268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,2,128,1,fp8,fp8,0,0.013023999333381654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,4,128,1,bfloat16,fp8,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,4,128,1,bfloat16,bfloat16,0,0.012302400171756744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,4,128,1,fp8,fp8,0,0.01313759982585907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,8,128,1,bfloat16,bfloat16,0,0.012100800126791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,8,128,1,bfloat16,fp8,0,0.013092799484729767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,8,128,1,fp8,fp8,0,0.012875199317932129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,96,128,1,bfloat16,bfloat16,0,0.01422560065984726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,96,128,1,bfloat16,fp8,0,0.013729600608348847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,96,128,1,fp8,fp8,0,0.014003199338912965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,1,128,1,fp8,fp8,0,0.012681600451469422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,1,128,1,bfloat16,fp8,0,0.012787200510501862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,2,128,1,bfloat16,bfloat16,0,0.01199520006775856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,2,128,1,bfloat16,fp8,0,0.01282079964876175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,2,128,1,fp8,fp8,0,0.01266240030527115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,4,128,1,bfloat16,bfloat16,0,0.012172800302505494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,4,128,1,bfloat16,fp8,0,0.012583999335765839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,4,128,1,fp8,fp8,0,0.012787200510501862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,8,128,1,bfloat16,bfloat16,0,0.012129600346088409
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,8,128,1,bfloat16,fp8,0,0.012768000364303589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,8,128,1,fp8,fp8,0,0.012775999307632447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,1,128,1,bfloat16,bfloat16,0,0.02109760046005249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,1,128,1,bfloat16,fp8,0,0.022567999362945557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,1,128,1,fp8,fp8,0,0.02274720072746277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,2,128,1,bfloat16,bfloat16,0,0.02809920012950897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,2,128,1,bfloat16,fp8,0,0.031150400638580322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,2,128,1,fp8,fp8,0,0.03115519881248474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,4,128,1,bfloat16,bfloat16,0,0.042691200971603394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,4,128,1,bfloat16,fp8,0,0.04866879880428314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,96,96,128,1,fp8,fp8,0,0.500377607345581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,4,128,1,fp8,fp8,0,0.048655998706817624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,8,128,1,bfloat16,bfloat16,0,0.07116640210151673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,8,128,1,bfloat16,fp8,0,0.08420799970626831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,96,8,128,1,fp8,fp8,0,0.08480640053749085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,96,128,1,bfloat16,bfloat16,0,0.3490096092224121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,96,128,1,bfloat16,fp8,0,0.43760318756103517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,1,128,1,bfloat16,bfloat16,0,0.015169599652290344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,1,128,1,bfloat16,fp8,0,0.015780800580978395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,8,128,1,bfloat16,bfloat16,0,0.01271200031042099
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,2,128,1,bfloat16,bfloat16,0,0.0184688001871109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,96,128,1,fp8,fp8,0,0.4366112232208252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,2,128,1,bfloat16,fp8,0,0.020216000080108643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,2,128,1,fp8,fp8,0,0.020160000026226043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,4,128,1,bfloat16,bfloat16,0,0.026238399744033813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,4,128,1,bfloat16,fp8,0,0.0290336012840271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,4,128,1,fp8,fp8,0,0.028968000411987306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,8,128,1,bfloat16,bfloat16,0,0.040833601355552675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,8,128,1,bfloat16,fp8,0,0.04664640128612518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,8,128,1,fp8,fp8,0,0.04664640128612518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,96,2,128,1,bfloat16,fp8,0,0.01327199935913086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,96,128,1,bfloat16,bfloat16,0,0.182697594165802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,1,128,1,bfloat16,bfloat16,0,0.01361120045185089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,96,128,1,bfloat16,fp8,0,0.22343840599060058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,1,128,1,bfloat16,fp8,0,0.01467359960079193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,1,128,1,fp8,fp8,0,0.014499199390411378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,2,128,1,bfloat16,fp8,0,0.014636799693107605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,2,128,1,fp8,fp8,0,0.01443839967250824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,4,128,1,bfloat16,bfloat16,0,0.01740480065345764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,4,128,1,bfloat16,fp8,0,0.018799999356269838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,96,1,128,1,bfloat16,bfloat16,0,0.011958400160074234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,4,128,1,fp8,fp8,0,0.018855999410152435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,8,128,1,bfloat16,bfloat16,0,0.02499520033597946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,8,128,1,fp8,fp8,0,0.027654400467872618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,96,128,1,bfloat16,bfloat16,0,0.09915040135383606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,96,128,1,bfloat16,fp8,0,0.1184656023979187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,1,128,1,bfloat16,bfloat16,0,0.012516799569129943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,1,128,1,bfloat16,fp8,0,0.013558399677276612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,1,128,1,fp8,fp8,0,0.013580800592899322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,2,128,1,bfloat16,bfloat16,0,0.012731200456619263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,2,128,1,bfloat16,fp8,0,0.01342879980802536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,2,128,1,fp8,fp8,0,0.013388800621032714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,4,128,1,bfloat16,bfloat16,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,4,128,1,bfloat16,fp8,0,0.013579200208187103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,4,128,1,fp8,fp8,0,0.013544000685214996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,8,128,1,bfloat16,bfloat16,0,0.016484799981117248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,8,128,1,bfloat16,fp8,0,0.017876799404621124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,8,128,1,fp8,fp8,0,0.01746399998664856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,96,128,1,bfloat16,bfloat16,0,0.054966402053833005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,96,128,1,bfloat16,fp8,0,0.06279360055923462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,96,128,1,fp8,fp8,0,0.06279839873313904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,1,128,1,bfloat16,bfloat16,0,0.012006399780511856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,1,128,1,bfloat16,fp8,0,0.012751999497413635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,1,128,1,fp8,fp8,0,0.01265760064125061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,2,128,1,bfloat16,bfloat16,0,0.01204639971256256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,96,4,128,1,fp8,fp8,0,0.013097600638866424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,2,128,1,bfloat16,fp8,0,0.012691199779510498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,96,1,128,1,fp8,fp8,0,0.01648000031709671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,2,128,1,fp8,fp8,0,0.012999999523162841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,4,128,1,bfloat16,bfloat16,0,0.012145599722862244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,4,128,1,bfloat16,fp8,0,0.01287200003862381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,4,128,1,fp8,fp8,0,0.01310880035161972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,8,128,1,bfloat16,bfloat16,0,0.012203200161457062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,8,128,1,bfloat16,fp8,0,0.013116799294948578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,96,8,128,1,fp8,fp8,0,0.013012799620628356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,96,128,1,bfloat16,bfloat16,0,0.03272800147533417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,96,128,1,bfloat16,fp8,0,0.036771199107170104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,96,128,1,fp8,fp8,0,0.036692801117897036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,1,128,1,bfloat16,fp8,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,1,128,1,fp8,fp8,0,0.012639999389648438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,2,128,1,bfloat16,bfloat16,0,0.011865600198507308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,2,128,1,bfloat16,fp8,0,0.01268479973077774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,2,128,1,fp8,fp8,0,0.012563200294971466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,4,128,1,bfloat16,bfloat16,0,0.012043199688196182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,4,128,1,bfloat16,fp8,0,0.012972800433635712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,96,128,1,fp8,fp8,0,0.22407999038696289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,4,128,1,fp8,fp8,0,0.012811200320720672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,8,128,1,bfloat16,bfloat16,0,0.012332800030708312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,8,128,1,bfloat16,fp8,0,0.012923200428485871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,8,128,1,fp8,fp8,0,0.012884800136089326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,2,128,1,bfloat16,bfloat16,0,0.013915200531482697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,96,128,1,bfloat16,bfloat16,0,0.021827200055122377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,96,128,1,bfloat16,fp8,0,0.022705599665641785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,96,128,1,fp8,fp8,0,0.022635200619697572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,1,128,1,bfloat16,bfloat16,0,0.01165120005607605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,1,128,1,bfloat16,fp8,0,0.0125231996178627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,1,128,1,fp8,fp8,0,0.012520000338554382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,2,128,1,bfloat16,fp8,0,0.012347199767827988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,2,128,1,fp8,fp8,0,0.012323199957609176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,4,128,1,bfloat16,bfloat16,0,0.01188800036907196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,4,128,1,bfloat16,fp8,0,0.01255359947681427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,4,128,1,fp8,fp8,0,0.012388800084590913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,8,128,1,bfloat16,fp8,0,0.012683199346065521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,8,128,1,fp8,fp8,0,0.012950399518013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,96,128,1,bfloat16,bfloat16,0,0.017894400656223296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,96,128,1,bfloat16,fp8,0,0.01805119961500168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,96,128,1,fp8,fp8,0,0.018199999630451203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,1,128,1,bfloat16,bfloat16,0,0.011627200245857238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,1,128,1,bfloat16,fp8,0,0.012409599870443344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,1,128,1,fp8,fp8,0,0.012470400333404541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,2,128,1,bfloat16,bfloat16,0,0.011689600348472596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,2,128,1,bfloat16,fp8,0,0.01249760016798973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,2,128,1,fp8,fp8,0,0.01239359974861145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,4,128,1,bfloat16,bfloat16,0,0.011779200285673141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,4,128,1,bfloat16,fp8,0,0.0123648002743721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,4,128,1,fp8,fp8,0,0.012439999729394913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,8,128,1,bfloat16,bfloat16,0,0.011699199676513672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,8,128,1,bfloat16,fp8,0,0.012720000743865967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,96,8,128,1,fp8,fp8,0,0.012399999797344208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,96,128,1,bfloat16,bfloat16,0,0.014095999300479889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,96,128,1,bfloat16,fp8,0,0.013689599931240082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,96,128,1,fp8,fp8,0,0.013808000087738036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,1,128,1,bfloat16,fp8,0,0.012436799705028534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,1,128,1,fp8,fp8,0,0.012412799894809723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,2,128,1,bfloat16,bfloat16,0,0.011619199812412263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,2,128,1,bfloat16,fp8,0,0.012323199957609176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,2,128,1,fp8,fp8,0,0.012518399953842163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,4,128,1,bfloat16,bfloat16,0,0.011675199866294861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,4,128,1,bfloat16,fp8,0,0.01233920007944107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,4,128,1,fp8,fp8,0,0.012342400103807449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,8,128,1,bfloat16,bfloat16,0,0.011615999788045884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,8,128,1,bfloat16,fp8,0,0.012336000055074691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,8,128,1,fp8,fp8,0,0.01231359988451004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,96,1,128,1,bfloat16,bfloat16,0,0.01180799975991249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,2,128,1,bfloat16,bfloat16,0,0.011711999773979187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,96,8,128,1,bfloat16,bfloat16,0,0.011873599886894227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,96,8,128,1,bfloat16,fp8,0,0.027537599205970764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,96,96,128,1,fp8,fp8,0,0.11876480579376221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,96,1,128,1,bfloat16,bfloat16,0,0.011556799709796905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,1,128,1,bfloat16,fp8,0,51.54674072265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,2,128,1,bfloat16,fp8,0,51.1286865234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,1,128,1,fp8,fp8,0,52.6382080078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,2,128,1,fp8,fp8,0,52.9001220703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,4,128,1,bfloat16,fp8,0,51.934130859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,1,128,1,bfloat16,bfloat16,0,60.249822998046874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,2,128,1,bfloat16,bfloat16,0,60.887451171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,4,128,1,bfloat16,bfloat16,0,61.10652465820313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,1,128,1,bfloat16,bfloat16,0,31.310092163085937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,4,128,1,fp8,fp8,0,51.51824951171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,8,128,1,bfloat16,fp8,0,52.985748291015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,8,128,1,fp8,fp8,0,53.105096435546876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,64,8,128,1,bfloat16,bfloat16,0,61.0899169921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,1,128,1,bfloat16,fp8,0,26.107467651367188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,1,128,1,fp8,fp8,0,26.084722900390624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,2,128,1,bfloat16,fp8,0,26.744180297851564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,2,128,1,bfloat16,bfloat16,0,31.255804443359374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,2,128,1,fp8,fp8,0,25.002561950683592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,4,128,1,bfloat16,bfloat16,0,30.382693481445312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,4,128,1,bfloat16,fp8,0,25.87732849121094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,4,128,1,fp8,fp8,0,26.92688903808594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,8,128,1,bfloat16,fp8,0,26.062588500976563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,8,128,1,bfloat16,bfloat16,0,32.00461120605469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,8,128,1,fp8,fp8,0,25.667019653320313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,1,128,1,bfloat16,bfloat16,0,15.200361633300782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,1,128,1,bfloat16,fp8,0,13.112887573242187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,1,128,1,fp8,fp8,0,13.418789672851563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,2,128,1,bfloat16,bfloat16,0,15.43736572265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,2,128,1,bfloat16,fp8,0,12.975674438476563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,2,128,1,fp8,fp8,0,12.408403015136718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,4,128,1,bfloat16,bfloat16,0,15.716941833496094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,4,128,1,bfloat16,fp8,0,13.07953338623047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,4,128,1,fp8,fp8,0,13.133172607421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,8,128,1,bfloat16,bfloat16,0,15.823321533203124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,64,128,1,fp8,fp8,0,12.723961639404298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,64,128,1,bfloat16,fp8,0,12.902627563476562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,8,128,1,bfloat16,fp8,0,12.843772888183594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,64,128,1,bfloat16,bfloat16,0,15.744473266601563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,64,128,1,bfloat16,fp8,0,6.4811859130859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,64,128,1,bfloat16,bfloat16,0,7.763540649414063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,64,8,128,1,fp8,fp8,0,12.537525177001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,64,128,1,bfloat16,fp8,0,25.675543212890624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,64,128,1,fp8,fp8,0,6.764289855957031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,1,128,1,bfloat16,bfloat16,0,7.216390228271484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,64,128,1,fp8,fp8,0,25.633502197265624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,1,128,1,bfloat16,fp8,0,6.2188255310058596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,1,128,1,fp8,fp8,0,6.229633712768555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,64,64,128,1,bfloat16,bfloat16,0,30.627490234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,2,128,1,bfloat16,fp8,0,6.29089126586914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,2,128,1,fp8,fp8,0,6.8229011535644535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,2,128,1,bfloat16,bfloat16,0,7.916595458984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,4,128,1,bfloat16,bfloat16,0,7.520811462402344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,4,128,1,bfloat16,fp8,0,6.327679824829102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,4,128,1,fp8,fp8,0,6.497723388671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,8,128,1,bfloat16,bfloat16,0,8.222774505615234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,8,128,1,bfloat16,fp8,0,7.215932464599609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,64,8,128,1,fp8,fp8,0,7.180155181884766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,1,128,1,bfloat16,fp8,0,29.323886108398437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,1,128,1,fp8,fp8,0,28.614456176757812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,2,128,1,bfloat16,fp8,0,29.34039306640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,2,128,1,fp8,fp8,0,28.26313171386719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,1,128,1,bfloat16,bfloat16,0,35.2351318359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,4,128,1,bfloat16,fp8,0,28.429718017578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,2,128,1,bfloat16,bfloat16,0,33.75699157714844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,4,128,1,bfloat16,bfloat16,0,34.31584167480469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,64,128,1,bfloat16,fp8,0,16.046636962890624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,64,128,1,fp8,fp8,0,15.986080932617188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,64,128,1,bfloat16,bfloat16,0,19.09727325439453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,1,128,1,bfloat16,bfloat16,0,17.39965057373047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,4,128,1,fp8,fp8,0,29.819976806640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,8,128,1,fp8,fp8,0,29.16907958984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,8,128,1,bfloat16,fp8,0,31.114364624023438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,64,8,128,1,bfloat16,bfloat16,0,35.45849609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,1,128,1,bfloat16,fp8,0,14.163482666015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,1,128,1,fp8,fp8,0,14.5751220703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,2,128,1,bfloat16,fp8,0,14.603959655761718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,2,128,1,bfloat16,bfloat16,0,16.884857177734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,2,128,1,fp8,fp8,0,14.458473205566406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,4,128,1,bfloat16,fp8,0,14.105439758300781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,4,128,1,fp8,fp8,0,14.814901733398438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,4,128,1,bfloat16,bfloat16,0,17.984036254882813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,8,128,1,bfloat16,fp8,0,15.210218811035157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,8,128,1,bfloat16,bfloat16,0,18.180931091308594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,64,128,1,bfloat16,bfloat16,0,9.592628479003906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,64,128,1,bfloat16,fp8,0,7.842100524902344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,64,128,1,fp8,fp8,0,7.355806732177735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,64,8,128,1,fp8,fp8,0,14.9409423828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,1,128,1,bfloat16,fp8,0,7.537786865234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,1,128,1,bfloat16,bfloat16,0,8.964981079101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,1,128,1,fp8,fp8,0,7.493599700927734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,2,128,1,fp8,fp8,0,7.037397003173828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,2,128,1,bfloat16,fp8,0,7.0775390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,2,128,1,bfloat16,bfloat16,0,8.346726226806641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,4,128,1,bfloat16,fp8,0,7.336080169677734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,4,128,1,bfloat16,bfloat16,0,8.593029022216797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,4,128,1,fp8,fp8,0,7.719435119628907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,64,128,1,bfloat16,fp8,0,3.764015960693359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,8,128,1,bfloat16,fp8,0,7.371769714355469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,64,128,1,bfloat16,bfloat16,0,4.524809646606445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,64,128,1,fp8,fp8,0,3.800902557373047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,8,128,1,bfloat16,bfloat16,0,8.566324615478516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,64,8,128,1,fp8,fp8,0,7.145449829101563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,1,128,1,bfloat16,bfloat16,0,4.794219207763672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,1,128,1,fp8,fp8,0,3.5023937225341797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,1,128,1,bfloat16,fp8,0,3.7530975341796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,2,128,1,bfloat16,bfloat16,0,4.285478210449218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,2,128,1,bfloat16,fp8,0,3.688790512084961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,2,128,1,fp8,fp8,0,3.5480289459228516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,4,128,1,bfloat16,bfloat16,0,4.327329635620117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,4,128,1,bfloat16,fp8,0,3.514231872558594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,4,128,1,fp8,fp8,0,4.038159942626953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,8,128,1,bfloat16,fp8,0,3.8415153503417967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,8,128,1,bfloat16,bfloat16,0,4.356177520751953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,64,8,128,1,fp8,fp8,0,3.7491519927978514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,1,128,1,bfloat16,fp8,0,20.660243225097656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,1,128,1,fp8,fp8,0,20.806320190429688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,2,128,1,bfloat16,fp8,0,19.714962768554688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,1,128,1,bfloat16,bfloat16,0,24.49254608154297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,4,128,1,bfloat16,fp8,0,21.11457061767578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,2,128,1,fp8,fp8,0,21.638583374023437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,2,128,1,bfloat16,bfloat16,0,24.367164611816406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,4,128,1,bfloat16,bfloat16,0,24.802645874023437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,64,128,1,bfloat16,fp8,0,11.210963439941406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,64,128,1,fp8,fp8,0,10.45621109008789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,64,128,1,bfloat16,bfloat16,0,13.364845275878906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,1,128,1,bfloat16,bfloat16,0,11.765542602539062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,4,128,1,fp8,fp8,0,20.639907836914062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,8,128,1,bfloat16,fp8,0,21.746400451660158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,8,128,1,fp8,fp8,0,20.409156799316406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,64,8,128,1,bfloat16,bfloat16,0,24.706210327148437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,1,128,1,bfloat16,fp8,0,10.606797027587891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,1,128,1,fp8,fp8,0,10.058328247070312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,2,128,1,bfloat16,bfloat16,0,12.29751205444336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,2,128,1,bfloat16,fp8,0,9.939630126953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,2,128,1,fp8,fp8,0,10.307982635498046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,4,128,1,bfloat16,fp8,0,10.022388458251953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,4,128,1,fp8,fp8,0,10.127986907958984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,4,128,1,bfloat16,bfloat16,0,11.709966278076172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,64,128,1,bfloat16,fp8,0,5.344379043579101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,64,128,1,bfloat16,bfloat16,0,6.596752166748047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,8,128,1,bfloat16,fp8,0,10.111592102050782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,8,128,1,bfloat16,bfloat16,0,12.622284698486329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,64,8,128,1,fp8,fp8,0,10.233857727050781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,1,128,1,bfloat16,fp8,0,4.974188613891601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,64,128,1,fp8,fp8,0,5.528297424316406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,1,128,1,bfloat16,bfloat16,0,6.311715316772461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,1,128,1,fp8,fp8,0,5.112673568725586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,2,128,1,bfloat16,bfloat16,0,6.038473510742188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,2,128,1,bfloat16,fp8,0,4.921353530883789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,2,128,1,fp8,fp8,0,5.5614879608154295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,4,128,1,bfloat16,fp8,0,4.947175979614258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,4,128,1,fp8,fp8,0,5.153654479980469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,4,128,1,bfloat16,bfloat16,0,6.511483001708984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,8,128,1,bfloat16,fp8,0,5.196204757690429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,8,128,1,bfloat16,bfloat16,0,5.996384048461914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,64,128,1,bfloat16,bfloat16,0,3.203878402709961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,64,8,128,1,fp8,fp8,0,4.988070297241211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,64,128,1,bfloat16,fp8,0,2.982881546020508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,64,128,1,fp8,fp8,0,2.668467140197754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,1,128,1,bfloat16,bfloat16,0,2.9186159133911134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,1,128,1,bfloat16,fp8,0,2.639015960693359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,1,128,1,fp8,fp8,0,2.5024736404418944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,2,128,1,bfloat16,bfloat16,0,2.972110366821289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,2,128,1,bfloat16,fp8,0,2.738849639892578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,2,128,1,fp8,fp8,0,2.466115188598633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,4,128,1,bfloat16,fp8,0,2.608568000793457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,4,128,1,fp8,fp8,0,2.660508728027344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,4,128,1,bfloat16,bfloat16,0,3.5678432464599608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,8,128,1,bfloat16,fp8,0,2.507339286804199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,8,128,1,bfloat16,bfloat16,0,3.050356864929199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,64,8,128,1,fp8,fp8,0,2.6289663314819336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,1,128,1,bfloat16,fp8,0,26.651260375976562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,1,128,1,fp8,fp8,0,26.918060302734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,2,128,1,bfloat16,fp8,0,26.97518310546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,2,128,1,fp8,fp8,0,26.969485473632812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,1,128,1,bfloat16,bfloat16,0,31.376025390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,4,128,1,bfloat16,fp8,0,25.730838012695312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,2,128,1,bfloat16,bfloat16,0,32.62637634277344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,4,128,1,bfloat16,bfloat16,0,31.83018798828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,64,128,1,bfloat16,bfloat16,0,16.75312194824219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,64,128,1,fp8,fp8,0,14.797373962402343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,64,128,1,bfloat16,fp8,0,15.444999694824219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,1,128,1,bfloat16,bfloat16,0,15.877168273925781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,4,128,1,fp8,fp8,0,26.491531372070312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,8,128,1,bfloat16,fp8,0,27.089129638671874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,8,128,1,fp8,fp8,0,27.561334228515626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,64,8,128,1,bfloat16,bfloat16,0,31.96924743652344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,1,128,1,bfloat16,fp8,0,13.089198303222656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,2,128,1,bfloat16,fp8,0,13.17530517578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,1,128,1,fp8,fp8,0,14.789389038085938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,2,128,1,bfloat16,bfloat16,0,15.483564758300782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,2,128,1,fp8,fp8,0,13.467880249023438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,4,128,1,bfloat16,fp8,0,13.613941955566407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,4,128,1,bfloat16,bfloat16,0,16.109080505371093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,4,128,1,fp8,fp8,0,13.261203002929687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,64,128,1,bfloat16,bfloat16,0,8.44309310913086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,64,128,1,bfloat16,fp8,0,7.013639831542969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,8,128,1,bfloat16,fp8,0,13.369345092773438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,8,128,1,bfloat16,bfloat16,0,15.9616943359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,64,128,1,fp8,fp8,0,7.348721313476562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,64,8,128,1,fp8,fp8,0,13.773776245117187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,1,128,1,bfloat16,bfloat16,0,7.868723297119141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,1,128,1,bfloat16,fp8,0,6.678966522216797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,1,128,1,fp8,fp8,0,6.689530944824218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,2,128,1,bfloat16,bfloat16,0,8.254821014404296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,2,128,1,bfloat16,fp8,0,6.697808074951172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,2,128,1,fp8,fp8,0,6.43463363647461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,4,128,1,bfloat16,fp8,0,6.969134521484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,4,128,1,bfloat16,bfloat16,0,8.102388763427735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,4,128,1,fp8,fp8,0,6.572583770751953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,8,128,1,bfloat16,fp8,0,6.823477172851563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,64,128,1,bfloat16,fp8,0,3.435454559326172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,8,128,1,bfloat16,bfloat16,0,7.999894714355468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,64,128,1,bfloat16,bfloat16,0,4.328329467773438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,64,128,1,fp8,fp8,0,3.6643985748291015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,64,8,128,1,fp8,fp8,0,6.5233100891113285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,1,128,1,bfloat16,bfloat16,0,4.03233757019043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,1,128,1,fp8,fp8,0,3.2812736511230467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,2,128,1,bfloat16,bfloat16,0,3.7678848266601563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,2,128,1,bfloat16,fp8,0,3.3631473541259767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,2,128,1,fp8,fp8,0,3.565327835083008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,4,128,1,bfloat16,fp8,0,3.2623294830322265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,4,128,1,bfloat16,bfloat16,0,3.9709278106689454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,4,128,1,fp8,fp8,0,3.4325809478759766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,1,128,1,bfloat16,fp8,0,3.2895503997802735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,64,128,1,bfloat16,bfloat16,0,1.9562448501586913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,64,128,1,bfloat16,fp8,0,1.7351104736328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,8,128,1,bfloat16,bfloat16,0,4.001577758789063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,64,128,1,fp8,fp8,0,1.926710319519043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,8,128,1,fp8,fp8,0,3.293017578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,1,128,1,bfloat16,fp8,0,1.6121824264526368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,1,128,1,bfloat16,bfloat16,0,1.835643196105957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,1,128,1,fp8,fp8,0,1.7868431091308594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,2,128,1,bfloat16,bfloat16,0,1.838270378112793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,2,128,1,bfloat16,fp8,0,1.6961856842041017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,2,128,1,fp8,fp8,0,1.7087583541870117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,4,128,1,bfloat16,fp8,0,1.6125743865966797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,4,128,1,fp8,fp8,0,1.7586944580078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,4,128,1,bfloat16,bfloat16,0,1.8214223861694336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,8,128,1,bfloat16,bfloat16,0,1.8358224868774413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,8,128,1,bfloat16,fp8,0,1.8542303085327148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,64,8,128,1,fp8,fp8,0,1.6299087524414062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,64,8,128,1,bfloat16,fp8,0,3.407571029663086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,1,128,1,bfloat16,fp8,0,14.878001403808593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,1,128,1,fp8,fp8,0,15.641941833496094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,2,128,1,bfloat16,fp8,0,14.902154541015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,2,128,1,fp8,fp8,0,15.208966064453126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,4,128,1,bfloat16,fp8,0,14.869830322265624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,1,128,1,bfloat16,bfloat16,0,18.37293395996094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,2,128,1,bfloat16,bfloat16,0,18.207705688476562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,4,128,1,bfloat16,bfloat16,0,18.048333740234376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,64,128,1,bfloat16,bfloat16,0,10.112427520751954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,64,128,1,bfloat16,fp8,0,8.464615631103516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,64,128,1,fp8,fp8,0,8.344334411621094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,1,128,1,bfloat16,bfloat16,0,9.139727783203124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,4,128,1,fp8,fp8,0,15.823110961914063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,8,128,1,bfloat16,fp8,0,15.93201141357422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,8,128,1,fp8,fp8,0,15.43223876953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,1,128,1,bfloat16,fp8,0,7.647752380371093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,64,8,128,1,bfloat16,bfloat16,0,19.061970520019532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,1,128,1,fp8,fp8,0,7.954156494140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,2,128,1,bfloat16,fp8,0,7.633537292480469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,2,128,1,bfloat16,bfloat16,0,9.302639770507813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,2,128,1,fp8,fp8,0,8.090547180175781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,4,128,1,bfloat16,fp8,0,7.83310546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,4,128,1,bfloat16,bfloat16,0,9.434180450439452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,64,128,1,bfloat16,bfloat16,0,5.029385757446289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,4,128,1,fp8,fp8,0,8.05299072265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,8,128,1,bfloat16,fp8,0,8.077161407470703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,64,128,1,bfloat16,fp8,0,4.066447830200195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,8,128,1,fp8,fp8,0,7.542625427246094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,64,8,128,1,bfloat16,bfloat16,0,9.237684631347657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,64,128,1,fp8,fp8,0,4.125630569458008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,1,128,1,bfloat16,bfloat16,0,4.475228881835937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,1,128,1,bfloat16,fp8,0,3.7212287902832033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,1,128,1,fp8,fp8,0,3.9425582885742188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,2,128,1,bfloat16,fp8,0,3.734960174560547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,2,128,1,bfloat16,bfloat16,0,4.410300827026367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,2,128,1,fp8,fp8,0,3.8346065521240233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,4,128,1,bfloat16,bfloat16,0,4.352967834472656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,4,128,1,bfloat16,fp8,0,3.844843292236328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,4,128,1,fp8,fp8,0,3.724848175048828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,64,128,1,bfloat16,bfloat16,0,2.4411136627197267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,64,128,1,bfloat16,fp8,0,2.315956878662109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,64,128,1,fp8,fp8,0,2.030678367614746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,8,128,1,bfloat16,fp8,0,3.8191009521484376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,8,128,1,bfloat16,bfloat16,0,4.664174270629883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,64,8,128,1,fp8,fp8,0,4.071807861328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,1,128,1,bfloat16,fp8,0,1.8646095275878907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,1,128,1,bfloat16,bfloat16,0,2.1493856430053713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,1,128,1,fp8,fp8,0,2.056801605224609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,2,128,1,bfloat16,fp8,0,1.8835359573364259
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,2,128,1,bfloat16,bfloat16,0,2.227894401550293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,2,128,1,fp8,fp8,0,1.8758800506591797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,4,128,1,bfloat16,fp8,0,1.8709823608398437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,4,128,1,bfloat16,bfloat16,0,2.330855941772461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,4,128,1,fp8,fp8,0,1.8633888244628907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,64,128,1,bfloat16,fp8,0,1.0761247634887696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,64,128,1,bfloat16,bfloat16,0,1.262183952331543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,8,128,1,bfloat16,fp8,0,1.8932672500610352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,8,128,1,bfloat16,bfloat16,0,2.3873519897460938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,64,8,128,1,fp8,fp8,0,1.8726783752441407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,1,128,1,bfloat16,fp8,0,0.9395263671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,64,128,1,fp8,fp8,0,1.023801612854004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,1,128,1,bfloat16,bfloat16,0,1.206385612487793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,1,128,1,fp8,fp8,0,0.9592384338378906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,2,128,1,bfloat16,bfloat16,0,1.1102272033691407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,2,128,1,bfloat16,fp8,0,0.968939208984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,2,128,1,fp8,fp8,0,1.0929391860961915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,4,128,1,bfloat16,bfloat16,0,1.0276880264282227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,4,128,1,bfloat16,fp8,0,0.9410320281982422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,4,128,1,fp8,fp8,0,0.951632022857666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,8,128,1,bfloat16,bfloat16,0,1.167684841156006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,8,128,1,bfloat16,fp8,0,0.9680656433105469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,64,8,128,1,fp8,fp8,0,1.0548720359802246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,1,128,1,fp8,fp8,0,13.995710754394532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,1,128,1,bfloat16,fp8,0,14.333598327636718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,2,128,1,bfloat16,fp8,0,14.503388977050781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,4,128,1,bfloat16,fp8,0,14.239361572265626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,2,128,1,fp8,fp8,0,14.962959289550781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,2,128,1,bfloat16,bfloat16,0,16.95929412841797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,1,128,1,bfloat16,bfloat16,0,17.10590515136719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,4,128,1,bfloat16,bfloat16,0,17.6052734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,64,128,1,bfloat16,fp8,0,8.026868438720703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,64,128,1,fp8,fp8,0,8.163694763183594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,64,128,1,bfloat16,bfloat16,0,9.959812927246094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,1,128,1,bfloat16,bfloat16,0,8.189363098144531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,4,128,1,fp8,fp8,0,14.7549072265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,8,128,1,bfloat16,fp8,0,14.540716552734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,8,128,1,fp8,fp8,0,14.439469909667968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,64,8,128,1,bfloat16,bfloat16,0,17.583367919921876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,1,128,1,bfloat16,fp8,0,6.9863533020019535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,1,128,1,fp8,fp8,0,7.227969360351563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,2,128,1,bfloat16,fp8,0,7.015386962890625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,2,128,1,bfloat16,bfloat16,0,8.758999633789063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,2,128,1,fp8,fp8,0,7.242876434326172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,4,128,1,bfloat16,fp8,0,7.366036987304687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,4,128,1,bfloat16,bfloat16,0,8.663494110107422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,4,128,1,fp8,fp8,0,7.188915252685547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,64,128,1,bfloat16,bfloat16,0,4.8636737823486325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,8,128,1,bfloat16,fp8,0,7.40259017944336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,64,128,1,bfloat16,fp8,0,3.98596305847168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,8,128,1,fp8,fp8,0,7.260104370117188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,64,8,128,1,bfloat16,bfloat16,0,8.715364837646485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,64,128,1,fp8,fp8,0,3.928902435302734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,1,128,1,bfloat16,bfloat16,0,4.172623825073242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,1,128,1,bfloat16,fp8,0,3.5034111022949217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,1,128,1,fp8,fp8,0,3.7927425384521483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,2,128,1,bfloat16,fp8,0,3.5010097503662108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,2,128,1,fp8,fp8,0,3.6594894409179686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,2,128,1,bfloat16,bfloat16,0,4.223763275146484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,4,128,1,bfloat16,fp8,0,3.5227489471435547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,4,128,1,bfloat16,bfloat16,0,4.044587326049805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,4,128,1,fp8,fp8,0,3.568369674682617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,64,128,1,bfloat16,bfloat16,0,2.3922399520874023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,8,128,1,bfloat16,bfloat16,0,4.38726577758789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,64,128,1,fp8,fp8,0,1.9683408737182617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,1,128,1,bfloat16,bfloat16,0,2.0307584762573243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,8,128,1,bfloat16,fp8,0,3.7528240203857424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,64,8,128,1,fp8,fp8,0,3.528852844238281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,1,128,1,bfloat16,fp8,0,1.7832656860351563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,1,128,1,fp8,fp8,0,2.053558349609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,2,128,1,bfloat16,bfloat16,0,2.0634592056274412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,2,128,1,fp8,fp8,0,1.872879981994629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,4,128,1,bfloat16,fp8,0,1.7664384841918945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,4,128,1,fp8,fp8,0,1.7805936813354493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,4,128,1,bfloat16,bfloat16,0,2.424608039855957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,64,128,1,bfloat16,fp8,0,2.1900720596313477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,64,128,1,bfloat16,bfloat16,0,1.1344544410705566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,8,128,1,bfloat16,bfloat16,0,1.9751567840576172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,8,128,1,fp8,fp8,0,1.7768304824829102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,64,128,1,bfloat16,fp8,0,1.1967408180236816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,8,128,1,bfloat16,fp8,0,1.9105552673339843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,64,128,1,fp8,fp8,0,0.9927295684814453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,1,128,1,bfloat16,bfloat16,0,1.0031071662902833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,1,128,1,bfloat16,fp8,0,1.0023088455200195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,1,128,1,fp8,fp8,0,0.9055071830749511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,2,128,1,bfloat16,fp8,0,0.892848014831543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,2,128,1,bfloat16,bfloat16,0,1.0097295761108398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,2,128,1,fp8,fp8,0,0.8840415954589844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,64,2,128,1,bfloat16,fp8,0,1.7573423385620117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,4,128,1,fp8,fp8,0,0.8872688293457032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,4,128,1,bfloat16,bfloat16,0,1.0049072265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,8,128,1,bfloat16,bfloat16,0,0.9673040390014649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,64,128,1,bfloat16,bfloat16,0,0.5688655853271485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,8,128,1,bfloat16,fp8,0,1.007369613647461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,8,128,1,fp8,fp8,0,0.891107177734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,64,128,1,bfloat16,fp8,0,0.5029056072235107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,64,128,1,fp8,fp8,0,0.5473616123199463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,1,128,1,bfloat16,bfloat16,0,0.5114160060882569
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,1,128,1,fp8,fp8,0,0.5206607818603516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,2,128,1,bfloat16,bfloat16,0,0.49608001708984373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,2,128,1,bfloat16,fp8,0,0.44753122329711914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,2,128,1,fp8,fp8,0,0.46681761741638184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,4,128,1,bfloat16,bfloat16,0,0.5143887996673584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,4,128,1,bfloat16,fp8,0,0.4844367980957031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,4,128,1,fp8,fp8,0,0.47912797927856443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,8,128,1,bfloat16,bfloat16,0,0.4794784069061279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,8,128,1,bfloat16,fp8,0,0.45542078018188475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,8,128,1,fp8,fp8,0,0.5012911796569824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,64,4,128,1,bfloat16,fp8,0,0.9455072402954101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,64,1,128,1,bfloat16,fp8,0,0.44795842170715333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,1,128,1,bfloat16,fp8,0,8.401513671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,1,128,1,fp8,fp8,0,8.44283218383789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,2,128,1,bfloat16,fp8,0,8.389524841308594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,1,128,1,bfloat16,bfloat16,0,10.141572570800781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,2,128,1,bfloat16,bfloat16,0,10.022557067871094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,2,128,1,fp8,fp8,0,8.297984313964843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,4,128,1,bfloat16,fp8,0,8.328447723388672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,4,128,1,bfloat16,bfloat16,0,9.743791961669922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,64,128,1,bfloat16,fp8,0,4.780174255371094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,64,128,1,bfloat16,bfloat16,0,6.120027160644531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,4,128,1,fp8,fp8,0,8.363944244384765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,64,128,1,fp8,fp8,0,4.786091232299805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,1,128,1,bfloat16,bfloat16,0,4.703219223022461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,8,128,1,bfloat16,fp8,0,8.782810974121094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,8,128,1,bfloat16,bfloat16,0,9.757041931152344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,64,8,128,1,fp8,fp8,0,9.184162902832032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,1,128,1,bfloat16,fp8,0,4.1466209411621096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,1,128,1,fp8,fp8,0,4.198195266723633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,2,128,1,bfloat16,fp8,0,4.158343887329101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,2,128,1,bfloat16,bfloat16,0,4.98231201171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,2,128,1,fp8,fp8,0,4.401694488525391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,4,128,1,bfloat16,fp8,0,4.188049697875977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,4,128,1,bfloat16,bfloat16,0,4.9491840362548825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,4,128,1,fp8,fp8,0,4.891854476928711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,64,128,1,bfloat16,fp8,0,2.4036144256591796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,64,128,1,bfloat16,bfloat16,0,2.8694719314575194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,8,128,1,bfloat16,fp8,0,4.199983978271485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,64,128,1,fp8,fp8,0,2.384359931945801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,1,128,1,bfloat16,bfloat16,0,2.4370752334594727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,8,128,1,fp8,fp8,0,4.201372909545898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,1,128,1,bfloat16,fp8,0,2.3293840408325197
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,1,128,1,fp8,fp8,0,2.083268737792969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,2,128,1,fp8,fp8,0,2.075035285949707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,2,128,1,bfloat16,fp8,0,2.077351951599121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,2,128,1,bfloat16,bfloat16,0,2.5256015777587892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,4,128,1,bfloat16,bfloat16,0,2.4956384658813477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,4,128,1,bfloat16,fp8,0,2.0867599487304687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,64,8,128,1,bfloat16,bfloat16,0,5.0006977081298825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,4,128,1,fp8,fp8,0,2.458937644958496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,8,128,1,bfloat16,bfloat16,0,2.307855987548828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,64,128,1,bfloat16,fp8,0,1.3649231910705566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,64,128,1,fp8,fp8,0,1.2093279838562012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,64,128,1,bfloat16,bfloat16,0,1.7802831649780273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,1,128,1,bfloat16,bfloat16,0,1.1075936317443849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,8,128,1,bfloat16,fp8,0,2.106929588317871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,64,8,128,1,fp8,fp8,0,2.1026752471923826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,1,128,1,bfloat16,fp8,0,1.0672752380371093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,1,128,1,fp8,fp8,0,1.194395160675049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,2,128,1,bfloat16,fp8,0,1.0434592247009278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,2,128,1,bfloat16,bfloat16,0,1.1734543800354005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,4,128,1,fp8,fp8,0,1.0507072448730468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,4,128,1,bfloat16,fp8,0,1.0576191902160645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,4,128,1,bfloat16,bfloat16,0,1.1758848190307618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,2,128,1,fp8,fp8,0,1.4372287750244142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,64,128,1,bfloat16,bfloat16,0,0.7338768005371094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,64,128,1,bfloat16,fp8,0,0.6086783885955811
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,8,128,1,bfloat16,bfloat16,0,1.5140640258789062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,8,128,1,bfloat16,fp8,0,1.1707023620605468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,1,128,1,bfloat16,bfloat16,0,0.5856783866882325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,64,8,128,1,fp8,fp8,0,1.0551695823669434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,1,128,1,bfloat16,fp8,0,0.6445151805877686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,2,128,1,bfloat16,bfloat16,0,0.5637919902801514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,2,128,1,bfloat16,fp8,0,0.5969711780548096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,2,128,1,fp8,fp8,0,0.5809919834136963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,4,128,1,bfloat16,bfloat16,0,0.5906943798065185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,4,128,1,bfloat16,fp8,0,0.5319808006286622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,4,128,1,fp8,fp8,0,0.6524608135223389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,8,128,1,bfloat16,bfloat16,0,0.5786511898040771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,64,128,1,bfloat16,fp8,0,0.31943039894104003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,8,128,1,fp8,fp8,0,0.5836160182952881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,64,128,1,bfloat16,bfloat16,0,0.3598207950592041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,8,128,1,bfloat16,fp8,0,0.665780782699585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,64,128,1,fp8,fp8,0,0.31917760372161863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,1,128,1,bfloat16,bfloat16,0,0.29524319171905516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,1,128,1,bfloat16,fp8,0,0.2836463928222656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,1,128,1,fp8,fp8,0,0.26905438899993894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,2,128,1,bfloat16,bfloat16,0,0.3267199993133545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,2,128,1,bfloat16,fp8,0,0.2817280054092407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,2,128,1,fp8,fp8,0,0.27441599369049074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,4,128,1,bfloat16,bfloat16,0,0.3011408090591431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,4,128,1,bfloat16,fp8,0,0.27068319320678713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,4,128,1,fp8,fp8,0,0.3032399892807007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,8,128,1,bfloat16,bfloat16,0,0.2989392042160034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,8,128,1,bfloat16,fp8,0,0.2746543884277344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,64,8,128,1,fp8,fp8,0,0.2827311992645264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,64,128,1,fp8,fp8,0,0.6111599922180175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,64,1,128,1,fp8,fp8,0,0.5990159988403321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,1,128,1,fp8,fp8,0,8.262907409667969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,2,128,1,bfloat16,fp8,0,8.24351806640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,1,128,1,bfloat16,fp8,0,8.763931274414062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,2,128,1,fp8,fp8,0,8.168833923339843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,2,128,1,bfloat16,bfloat16,0,9.6597900390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,1,128,1,bfloat16,bfloat16,0,10.214667510986327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,4,128,1,bfloat16,bfloat16,0,9.372576141357422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,64,128,1,bfloat16,fp8,0,4.896353530883789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,4,128,1,bfloat16,fp8,0,8.199320220947266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,64,128,1,bfloat16,bfloat16,0,5.975318527221679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,64,128,1,fp8,fp8,0,4.897310256958008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,4,128,1,fp8,fp8,0,8.249124908447266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,8,128,1,bfloat16,fp8,0,9.081171417236328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,8,128,1,fp8,fp8,0,8.255345916748047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,64,8,128,1,bfloat16,bfloat16,0,9.568863677978516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,1,128,1,bfloat16,fp8,0,4.078019332885742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,1,128,1,bfloat16,bfloat16,0,4.506633758544922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,1,128,1,fp8,fp8,0,4.089353561401367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,2,128,1,bfloat16,fp8,0,4.120979309082031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,2,128,1,bfloat16,bfloat16,0,4.713768005371094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,2,128,1,fp8,fp8,0,4.334475326538086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,4,128,1,bfloat16,fp8,0,4.111966323852539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,4,128,1,bfloat16,bfloat16,0,4.673907089233398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,4,128,1,fp8,fp8,0,4.100873565673828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,64,128,1,bfloat16,bfloat16,0,2.8983455657958985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,8,128,1,bfloat16,fp8,0,4.147806549072266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,8,128,1,bfloat16,bfloat16,0,4.85906867980957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,64,128,1,bfloat16,fp8,0,2.5373552322387694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,64,128,1,fp8,fp8,0,2.4398704528808595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,1,128,1,bfloat16,bfloat16,0,2.2353231430053713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,64,8,128,1,fp8,fp8,0,4.164518356323242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,1,128,1,bfloat16,fp8,0,2.041561508178711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,1,128,1,fp8,fp8,0,2.1042943954467774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,2,128,1,bfloat16,bfloat16,0,2.178107261657715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,2,128,1,bfloat16,fp8,0,2.0954879760742187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,2,128,1,fp8,fp8,0,2.120147132873535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,4,128,1,bfloat16,fp8,0,2.1150287628173827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,4,128,1,bfloat16,bfloat16,0,2.3048688888549806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,4,128,1,fp8,fp8,0,2.0559600830078124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,8,128,1,bfloat16,bfloat16,0,2.3575519561767577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,64,128,1,bfloat16,bfloat16,0,1.4284735679626466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,1,128,1,bfloat16,fp8,0,1.0249504089355468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,8,128,1,bfloat16,fp8,0,2.120284843444824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,64,128,1,fp8,fp8,0,1.2285375595092773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,64,128,1,bfloat16,fp8,0,1.2778752326965332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,64,8,128,1,fp8,fp8,0,2.3064319610595705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,1,128,1,fp8,fp8,0,1.1528304100036622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,2,128,1,bfloat16,fp8,0,1.0261679649353028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,2,128,1,bfloat16,bfloat16,0,1.1534223556518555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,2,128,1,fp8,fp8,0,1.0431232452392578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,4,128,1,bfloat16,bfloat16,0,1.0819760322570802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,4,128,1,bfloat16,fp8,0,1.0479647636413574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,4,128,1,fp8,fp8,0,1.1062416076660155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,64,128,1,bfloat16,bfloat16,0,0.7223360061645507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,64,128,1,bfloat16,fp8,0,0.6798624038696289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,64,128,1,fp8,fp8,0,0.636843204498291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,8,128,1,bfloat16,bfloat16,0,1.1454367637634277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,8,128,1,bfloat16,fp8,0,1.0424367904663085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,8,128,1,fp8,fp8,0,1.1336480140686036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,1,128,1,bfloat16,bfloat16,0,0.5573791980743408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,1,128,1,bfloat16,fp8,0,0.5221695899963379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,2,128,1,bfloat16,bfloat16,0,0.5543839931488037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,2,128,1,bfloat16,fp8,0,0.5403823852539062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,2,128,1,fp8,fp8,0,0.5327888011932373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,4,128,1,bfloat16,fp8,0,0.5633647918701172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,4,128,1,bfloat16,bfloat16,0,0.5775248050689697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,4,128,1,fp8,fp8,0,0.5239727973937989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,8,128,1,bfloat16,bfloat16,0,0.5717999935150146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,8,128,1,bfloat16,fp8,0,0.5469999790191651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,8,128,1,fp8,fp8,0,0.5409215927124024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,64,128,1,bfloat16,fp8,0,0.33179678916931155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,64,128,1,bfloat16,bfloat16,0,0.38698880672454833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,64,1,128,1,bfloat16,bfloat16,0,1.2372336387634277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,64,128,1,fp8,fp8,0,0.32080318927764895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,1,128,1,bfloat16,fp8,0,0.2734672069549561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,1,128,1,fp8,fp8,0,0.2717983961105347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,2,128,1,bfloat16,bfloat16,0,0.28840639591217043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,2,128,1,bfloat16,fp8,0,0.2894992113113403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,4,128,1,bfloat16,bfloat16,0,0.2824064016342163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,2,128,1,fp8,fp8,0,0.27256639003753663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,4,128,1,bfloat16,fp8,0,0.2733680009841919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,4,128,1,fp8,fp8,0,0.27413918972015383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,8,128,1,bfloat16,bfloat16,0,0.30762720108032227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,8,128,1,bfloat16,fp8,0,0.27022879123687743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,64,128,1,bfloat16,bfloat16,0,0.19568480253219606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,8,128,1,fp8,fp8,0,0.2756511926651001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,64,128,1,fp8,fp8,0,0.17932800054550171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,1,128,1,bfloat16,bfloat16,0,0.1484287977218628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,1,128,1,fp8,fp8,0,0.14130400419235228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,2,128,1,bfloat16,bfloat16,0,0.15875040292739867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,2,128,1,bfloat16,fp8,0,0.1419551968574524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,2,128,1,fp8,fp8,0,0.14194719791412352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,4,128,1,bfloat16,bfloat16,0,0.16015199422836304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,4,128,1,bfloat16,fp8,0,0.14264479875564576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,4,128,1,fp8,fp8,0,0.14289599657058716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,8,128,1,bfloat16,bfloat16,0,0.1705072045326233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,8,128,1,bfloat16,fp8,0,0.14428479671478273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,8,128,1,fp8,fp8,0,0.1443711996078491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,64,1,128,1,fp8,fp8,0,0.5653359889984131
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,64,1,128,1,bfloat16,bfloat16,0,0.29244160652160645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,64,128,1,bfloat16,fp8,0,0.1705407977104187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,64,1,128,1,bfloat16,fp8,0,0.14169280529022216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,1,128,1,bfloat16,fp8,0,5.12346076965332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,1,128,1,bfloat16,bfloat16,0,5.93636474609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,2,128,1,bfloat16,bfloat16,0,5.48533935546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,2,128,1,bfloat16,fp8,0,5.07423210144043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,2,128,1,fp8,fp8,0,5.050983810424805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,4,128,1,bfloat16,bfloat16,0,5.669889450073242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,1,128,1,fp8,fp8,0,5.045291137695313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,4,128,1,bfloat16,fp8,0,5.072663879394531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,4,128,1,fp8,fp8,0,5.085916900634766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,64,128,1,bfloat16,bfloat16,0,3.730748748779297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,8,128,1,bfloat16,fp8,0,5.182302474975586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,64,128,1,bfloat16,fp8,0,3.182111930847168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,8,128,1,bfloat16,bfloat16,0,5.697619247436523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,64,128,1,fp8,fp8,0,3.1685680389404296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,64,8,128,1,fp8,fp8,0,5.131052780151367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,1,128,1,bfloat16,bfloat16,0,2.7546560287475588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,1,128,1,bfloat16,fp8,0,2.5229183197021485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,1,128,1,fp8,fp8,0,2.650271987915039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,2,128,1,bfloat16,bfloat16,0,2.7936607360839845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,2,128,1,fp8,fp8,0,2.5344575881958007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,4,128,1,bfloat16,bfloat16,0,2.6876527786254885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,4,128,1,bfloat16,fp8,0,2.541187286376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,4,128,1,fp8,fp8,0,2.5899423599243163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,8,128,1,bfloat16,bfloat16,0,3.003041648864746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,64,128,1,bfloat16,fp8,0,1.6618640899658204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,64,128,1,bfloat16,bfloat16,0,1.8232383728027344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,64,128,1,fp8,fp8,0,1.5883775711059571
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,8,128,1,bfloat16,fp8,0,2.5762367248535156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,1,128,1,bfloat16,bfloat16,0,1.3087712287902833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,8,128,1,fp8,fp8,0,2.7268768310546876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,64,2,128,1,bfloat16,fp8,0,2.5194944381713866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,1,128,1,fp8,fp8,0,1.3013808250427246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,1,128,1,bfloat16,fp8,0,1.3619711875915528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,2,128,1,bfloat16,fp8,0,1.2672847747802733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,2,128,1,bfloat16,bfloat16,0,1.344491195678711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,2,128,1,fp8,fp8,0,1.270644760131836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,4,128,1,bfloat16,bfloat16,0,1.380998420715332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,4,128,1,bfloat16,fp8,0,1.2780608177185058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,64,128,1,bfloat16,bfloat16,0,0.9456735610961914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,4,128,1,fp8,fp8,0,1.27871675491333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,64,128,1,bfloat16,fp8,0,0.8009743690490723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,8,128,1,bfloat16,fp8,0,1.2947792053222655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,8,128,1,bfloat16,bfloat16,0,1.379246425628662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,64,8,128,1,fp8,fp8,0,1.2944016456604004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,1,128,1,bfloat16,bfloat16,0,0.6585375785827636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,64,128,1,fp8,fp8,0,0.8522895812988281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,1,128,1,fp8,fp8,0,0.6407055854797363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,2,128,1,bfloat16,bfloat16,0,0.6633008003234864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,1,128,1,bfloat16,fp8,0,0.6608751773834228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,2,128,1,bfloat16,fp8,0,0.6413407802581788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,2,128,1,fp8,fp8,0,0.6664127826690673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,4,128,1,bfloat16,bfloat16,0,0.6763567924499512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,4,128,1,bfloat16,fp8,0,0.651196813583374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,4,128,1,fp8,fp8,0,0.6549088001251221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,8,128,1,bfloat16,fp8,0,0.6546527862548828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,1,128,1,bfloat16,bfloat16,0,0.3501471996307373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,1,128,1,bfloat16,fp8,0,0.3259407997131348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,64,128,1,fp8,fp8,0,0.4148575782775879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,1,128,1,fp8,fp8,0,0.3259984016418457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,2,128,1,bfloat16,fp8,0,0.3250511884689331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,2,128,1,fp8,fp8,0,0.33194398880004883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,4,128,1,bfloat16,bfloat16,0,0.3410128116607666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,4,128,1,bfloat16,fp8,0,0.32693440914154054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,4,128,1,fp8,fp8,0,0.3336047887802124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,8,128,1,bfloat16,bfloat16,0,0.3536704063415527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,8,128,1,fp8,fp8,0,0.33352160453796387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,64,128,1,bfloat16,bfloat16,0,0.2469007968902588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,64,128,1,bfloat16,fp8,0,0.21239519119262695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,64,128,1,fp8,fp8,0,0.21635520458221436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,1,128,1,bfloat16,bfloat16,0,0.17373919486999512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,1,128,1,bfloat16,fp8,0,0.17176640033721924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,2,128,1,bfloat16,bfloat16,0,0.17888959646224975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,2,128,1,bfloat16,fp8,0,0.17214879989624024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,2,128,1,fp8,fp8,0,0.17202240228652954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,4,128,1,bfloat16,bfloat16,0,0.18091200590133666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,8,128,1,bfloat16,bfloat16,0,0.6902128219604492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,64,8,128,1,fp8,fp8,0,0.6678207874298095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,64,128,1,bfloat16,fp8,0,0.4135183811187744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,4,128,1,bfloat16,fp8,0,0.17443039417266845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,4,128,1,fp8,fp8,0,0.17408959865570067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,8,128,1,bfloat16,bfloat16,0,0.18086559772491456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,2,128,1,bfloat16,bfloat16,0,0.3450671911239624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,8,128,1,bfloat16,fp8,0,0.17615360021591187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,8,128,1,fp8,fp8,0,0.1759503960609436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,64,128,1,bfloat16,bfloat16,0,0.1279312014579773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,64,128,1,fp8,fp8,0,0.11487040519714356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,64,128,1,bfloat16,fp8,0,0.11383680105209351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,1,128,1,bfloat16,fp8,0,0.08906239867210389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,1,128,1,fp8,fp8,0,0.09128640294075012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,2,128,1,bfloat16,bfloat16,0,0.09579200148582459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,2,128,1,bfloat16,fp8,0,0.08924959897994995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,2,128,1,fp8,fp8,0,0.09186720252037048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,4,128,1,bfloat16,bfloat16,0,0.0965712010860443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,4,128,1,bfloat16,fp8,0,0.09045760035514831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,4,128,1,fp8,fp8,0,0.09143199920654296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,8,128,1,bfloat16,fp8,0,0.3310688018798828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,8,128,1,bfloat16,bfloat16,0,0.09828479886054993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,8,128,1,bfloat16,fp8,0,0.0917743980884552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,8,128,1,fp8,fp8,0,0.0925935983657837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,64,1,128,1,fp8,fp8,0,0.1679263949394226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,64,64,128,1,bfloat16,bfloat16,0,0.47285118103027346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,64,1,128,1,bfloat16,bfloat16,0,0.0958624005317688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,1,128,1,bfloat16,fp8,0,5.357788848876953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,1,128,1,fp8,fp8,0,5.3375999450683596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,1,128,1,bfloat16,bfloat16,0,5.668080139160156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,2,128,1,bfloat16,bfloat16,0,5.681428909301758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,2,128,1,bfloat16,fp8,0,5.3168384552001955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,2,128,1,fp8,fp8,0,5.35699691772461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,4,128,1,bfloat16,bfloat16,0,5.727638244628906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,4,128,1,bfloat16,fp8,0,5.3366657257080075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,64,128,1,bfloat16,bfloat16,0,4.197531127929688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,4,128,1,fp8,fp8,0,5.2748046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,8,128,1,bfloat16,fp8,0,5.397198486328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,8,128,1,fp8,fp8,0,5.394638442993164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,64,8,128,1,bfloat16,bfloat16,0,6.103081512451172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,64,128,1,bfloat16,fp8,0,3.5057632446289064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,64,128,1,fp8,fp8,0,3.4908065795898438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,1,128,1,bfloat16,bfloat16,0,2.7702896118164064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,1,128,1,bfloat16,fp8,0,2.7002304077148436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,1,128,1,fp8,fp8,0,2.620144081115723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,2,128,1,bfloat16,bfloat16,0,2.8272287368774416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,2,128,1,bfloat16,fp8,0,2.6359535217285157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,2,128,1,fp8,fp8,0,2.776272010803223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,4,128,1,bfloat16,bfloat16,0,2.7949600219726562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,4,128,1,bfloat16,fp8,0,2.642633628845215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,4,128,1,fp8,fp8,0,2.6699600219726562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,64,128,1,bfloat16,fp8,0,1.7589712142944336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,8,128,1,bfloat16,bfloat16,0,2.960807991027832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,1,128,1,bfloat16,bfloat16,0,1.349619197845459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,8,128,1,fp8,fp8,0,2.763907241821289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,1,128,1,bfloat16,fp8,0,1.3273088455200195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,1,128,1,fp8,fp8,0,1.3282032012939453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,2,128,1,bfloat16,fp8,0,1.3187775611877441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,2,128,1,bfloat16,bfloat16,0,1.4322896003723145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,2,128,1,fp8,fp8,0,1.3299391746520997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,4,128,1,bfloat16,bfloat16,0,1.3841327667236327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,4,128,1,bfloat16,fp8,0,1.3273903846740722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,4,128,1,fp8,fp8,0,1.3275823593139648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,8,128,1,bfloat16,bfloat16,0,1.5206368446350098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,64,128,1,bfloat16,bfloat16,0,2.066752052307129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,8,128,1,bfloat16,fp8,0,1.3613439559936524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,64,128,1,fp8,fp8,0,1.838630485534668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,64,128,1,bfloat16,bfloat16,0,1.0400704383850097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,64,8,128,1,fp8,fp8,0,1.3640239715576172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,64,128,1,bfloat16,fp8,0,0.8791824340820312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,1,128,1,bfloat16,fp8,0,0.6658400058746338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,1,128,1,bfloat16,bfloat16,0,0.6807712078094482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,64,128,1,fp8,fp8,0,0.9166208267211914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,1,128,1,fp8,fp8,0,0.6713903903961181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,2,128,1,bfloat16,bfloat16,0,0.6830512046813965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,2,128,1,bfloat16,fp8,0,0.6699440002441406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,2,128,1,fp8,fp8,0,0.66353759765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,4,128,1,bfloat16,bfloat16,0,0.6964687824249267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,4,128,1,bfloat16,fp8,0,0.6914671897888184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,8,128,1,bfloat16,bfloat16,0,0.7299856185913086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,64,128,1,bfloat16,bfloat16,0,0.5258607864379883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,8,128,1,fp8,fp8,0,0.7001776218414306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,8,128,1,bfloat16,fp8,0,0.7002624034881592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,64,128,1,bfloat16,fp8,0,0.4503200054168701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,64,128,1,fp8,fp8,0,0.4574751853942871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,1,128,1,bfloat16,bfloat16,0,0.3519056081771851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,1,128,1,bfloat16,fp8,0,0.33716320991516113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,1,128,1,fp8,fp8,0,0.3468559980392456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,2,128,1,bfloat16,bfloat16,0,0.35719521045684816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,2,128,1,bfloat16,fp8,0,0.3393392086029053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,2,128,1,fp8,fp8,0,0.3493488073348999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,4,128,1,bfloat16,bfloat16,0,0.360752010345459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,4,128,1,bfloat16,fp8,0,0.3404975891113281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,4,128,1,fp8,fp8,0,0.34995360374450685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,8,128,1,bfloat16,bfloat16,0,0.3748496055603027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,8,128,1,bfloat16,fp8,0,0.3481775999069214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,64,128,1,bfloat16,fp8,0,0.2300800085067749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,64,8,128,1,fp8,fp8,0,0.3585632085800171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,64,128,1,fp8,fp8,0,0.2348912000656128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,1,128,1,bfloat16,bfloat16,0,0.18472640514373778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,1,128,1,bfloat16,fp8,0,0.17775360345840455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,1,128,1,fp8,fp8,0,0.1740928053855896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,2,128,1,bfloat16,bfloat16,0,0.18598400354385375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,64,8,128,1,bfloat16,fp8,0,2.6883615493774413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,2,128,1,bfloat16,fp8,0,0.1798192024230957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,4,128,1,bfloat16,bfloat16,0,0.18228319883346558
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,4,128,1,bfloat16,fp8,0,0.18226399421691894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,8,128,1,bfloat16,bfloat16,0,0.19406399726867676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,4,128,1,fp8,fp8,0,0.17763040065765381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,8,128,1,bfloat16,fp8,0,0.17842719554901124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,8,128,1,fp8,fp8,0,0.18518879413604736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,64,128,1,fp8,fp8,0,0.12117120027542114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,64,128,1,bfloat16,bfloat16,0,0.14545439481735228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,64,128,1,bfloat16,fp8,0,0.12269279956817628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,1,128,1,bfloat16,fp8,0,0.09541280269622802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,1,128,1,fp8,fp8,0,0.0952400028705597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,2,128,1,bfloat16,bfloat16,0,0.09737600088119507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,2,128,1,bfloat16,fp8,0,0.0962831974029541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,2,128,1,fp8,fp8,0,0.0956816017627716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,4,128,1,bfloat16,bfloat16,0,0.09833279848098755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,4,128,1,bfloat16,fp8,0,0.09661759734153748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,4,128,1,fp8,fp8,0,0.09657760262489319
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,8,128,1,bfloat16,bfloat16,0,0.10088000297546387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,8,128,1,bfloat16,fp8,0,0.09823039770126343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,64,128,1,bfloat16,bfloat16,0,0.07686240077018738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,8,128,1,fp8,fp8,0,0.09789440035820007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,64,128,1,bfloat16,fp8,0,0.0671455979347229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,64,128,1,fp8,fp8,0,0.06630240082740783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,1,128,1,bfloat16,bfloat16,0,0.050779199600219725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,1,128,1,bfloat16,fp8,0,0.04986560046672821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,64,4,128,1,fp8,fp8,0,0.6849728107452393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,1,128,1,fp8,fp8,0,0.04887360036373138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,2,128,1,bfloat16,bfloat16,0,0.05168799757957458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,2,128,1,bfloat16,fp8,0,0.05011839866638183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,2,128,1,fp8,fp8,0,0.0484688013792038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,4,128,1,bfloat16,fp8,0,0.050183999538421634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,8,128,1,bfloat16,bfloat16,0,0.05621119737625122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,8,128,1,bfloat16,fp8,0,0.050918400287628174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,8,128,1,fp8,fp8,0,0.04971039891242981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,64,128,1,bfloat16,bfloat16,0,0.2750272035598755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,64,2,128,1,fp8,fp8,0,0.18015040159225465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,64,1,128,1,bfloat16,bfloat16,0,0.09900799989700318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,4,128,1,bfloat16,bfloat16,0,0.05277119874954224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,64,4,128,1,fp8,fp8,0,0.0502128005027771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,1,128,1,bfloat16,bfloat16,0,3.881097412109375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,1,128,1,bfloat16,fp8,0,3.848110580444336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,1,128,1,fp8,fp8,0,3.889726257324219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,2,128,1,bfloat16,bfloat16,0,3.9448238372802735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,2,128,1,bfloat16,fp8,0,3.8628734588623046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,2,128,1,fp8,fp8,0,3.860163116455078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,4,128,1,bfloat16,bfloat16,0,4.002267074584961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,4,128,1,bfloat16,fp8,0,3.905611038208008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,4,128,1,fp8,fp8,0,3.885065460205078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,8,128,1,bfloat16,bfloat16,0,4.191974258422851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,64,128,1,bfloat16,bfloat16,0,3.326824188232422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,8,128,1,bfloat16,fp8,0,3.990691375732422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,64,8,128,1,fp8,fp8,0,4.0126190185546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,1,128,1,bfloat16,bfloat16,0,1.932753562927246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,1,128,1,bfloat16,fp8,0,1.9232608795166015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,64,128,1,bfloat16,fp8,0,2.791904067993164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,64,128,1,fp8,fp8,0,2.8183984756469727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,1,128,1,fp8,fp8,0,1.927457618713379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,2,128,1,bfloat16,bfloat16,0,1.9863744735717774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,2,128,1,bfloat16,fp8,0,1.9180368423461913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,2,128,1,fp8,fp8,0,1.921660804748535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,4,128,1,bfloat16,bfloat16,0,2.0069536209106444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,4,128,1,bfloat16,fp8,0,1.9490064620971679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,4,128,1,fp8,fp8,0,1.9401039123535155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,8,128,1,bfloat16,bfloat16,0,2.0848079681396485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,64,128,1,bfloat16,fp8,0,1.4039119720458983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,1,128,1,bfloat16,bfloat16,0,0.9676447868347168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,64,128,1,bfloat16,bfloat16,0,1.67083683013916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,1,128,1,bfloat16,fp8,0,0.9625328063964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,64,128,1,fp8,fp8,0,1.4160719871520997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,1,128,1,fp8,fp8,0,0.9681360244750976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,2,128,1,bfloat16,bfloat16,0,0.9736656188964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,2,128,1,bfloat16,fp8,0,0.9744383811950683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,2,128,1,fp8,fp8,0,0.965401554107666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,4,128,1,bfloat16,bfloat16,0,0.9967519760131835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,4,128,1,bfloat16,fp8,0,0.9860527992248536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,4,128,1,fp8,fp8,0,0.9766927719116211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,8,128,1,bfloat16,bfloat16,0,1.0447808265686036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,8,128,1,bfloat16,fp8,0,1.007038402557373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,64,128,1,bfloat16,bfloat16,0,0.8432160377502441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,64,128,1,bfloat16,fp8,0,0.7146959781646729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,64,128,1,fp8,fp8,0,0.708732795715332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,64,8,128,1,fp8,fp8,0,1.0270208358764648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,1,128,1,bfloat16,bfloat16,0,0.4881120204925537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,1,128,1,bfloat16,fp8,0,0.4891088008880615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,2,128,1,bfloat16,bfloat16,0,0.4966464042663574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,1,128,1,fp8,fp8,0,0.48629279136657716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,2,128,1,bfloat16,fp8,0,0.48912639617919923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,2,128,1,fp8,fp8,0,0.4996511936187744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,4,128,1,bfloat16,bfloat16,0,0.5061552047729492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,4,128,1,bfloat16,fp8,0,0.4968544006347656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,4,128,1,fp8,fp8,0,0.4991727828979492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,8,128,1,bfloat16,fp8,0,2.002599906921387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,8,128,1,bfloat16,bfloat16,0,0.5282127857208252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,8,128,1,bfloat16,fp8,0,0.5081456184387207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,64,128,1,bfloat16,bfloat16,0,0.43042721748352053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,64,8,128,1,fp8,fp8,0,2.026728057861328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,64,128,1,bfloat16,fp8,0,0.36248319149017333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,64,128,1,fp8,fp8,0,0.3643631935119629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,1,128,1,bfloat16,bfloat16,0,0.25339999198913576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,2,128,1,bfloat16,fp8,0,0.25085279941558836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,2,128,1,fp8,fp8,0,0.2517904043197632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,4,128,1,bfloat16,bfloat16,0,0.25989599227905275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,4,128,1,bfloat16,fp8,0,0.2533519983291626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,4,128,1,fp8,fp8,0,0.2543519973754883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,8,128,1,bfloat16,bfloat16,0,0.271396803855896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,8,128,1,bfloat16,fp8,0,0.2602207899093628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,8,128,1,fp8,fp8,0,0.2613215923309326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,64,128,1,bfloat16,bfloat16,0,0.22327840328216553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,64,128,1,bfloat16,fp8,0,0.18776960372924806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,1,128,1,bfloat16,bfloat16,0,0.13167840242385864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,64,128,1,fp8,fp8,0,0.18838399648666382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,1,128,1,bfloat16,fp8,0,0.1302639961242676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,1,128,1,fp8,fp8,0,0.13089920282363893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,2,128,1,bfloat16,bfloat16,0,0.13299520015716554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,2,128,1,bfloat16,fp8,0,0.13123999834060668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,2,128,1,fp8,fp8,0,0.13159359693527223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,4,128,1,bfloat16,bfloat16,0,0.13556480407714844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,4,128,1,bfloat16,fp8,0,0.13252960443496703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,4,128,1,fp8,fp8,0,0.1327440023422241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,8,128,1,bfloat16,fp8,0,0.13519359827041627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,8,128,1,fp8,fp8,0,0.1357983946800232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,64,128,1,bfloat16,bfloat16,0,0.11927679777145386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,64,128,1,bfloat16,fp8,0,0.1004639983177185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,64,128,1,fp8,fp8,0,0.10119359493255616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,1,128,1,bfloat16,bfloat16,0,0.07175840139389038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,1,128,1,bfloat16,fp8,0,0.07067840099334717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,1,128,1,fp8,fp8,0,0.07096319794654846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,2,128,1,bfloat16,bfloat16,0,0.07200800180435181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,2,128,1,bfloat16,fp8,0,0.0715727984905243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,2,128,1,fp8,fp8,0,0.07103360295295716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,4,128,1,bfloat16,bfloat16,0,0.07314559817314148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,64,8,128,1,fp8,fp8,0,0.5176688194274902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,4,128,1,bfloat16,fp8,0,0.07193920016288757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,4,128,1,fp8,fp8,0,0.07212799787521362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,8,128,1,bfloat16,bfloat16,0,0.07634720206260681
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,8,128,1,fp8,fp8,0,0.0735264003276825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,64,128,1,bfloat16,bfloat16,0,0.06476799845695495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,1,128,1,fp8,fp8,0,0.2487567901611328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,64,128,1,bfloat16,fp8,0,0.056651198863983156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,64,128,1,fp8,fp8,0,0.057520002126693726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,1,128,1,bfloat16,bfloat16,0,0.03699840009212494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,1,128,1,bfloat16,fp8,0,0.037360000610351565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,1,128,1,fp8,fp8,0,0.03752639889717102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,2,128,1,bfloat16,bfloat16,0,0.0382095992565155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,2,128,1,bfloat16,bfloat16,0,0.2541280031204224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,2,128,1,bfloat16,fp8,0,0.037299200892448425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,2,128,1,fp8,fp8,0,0.03779839873313904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,4,128,1,bfloat16,bfloat16,0,0.039001598954200745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,4,128,1,bfloat16,fp8,0,0.03821280002593994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,8,128,1,bfloat16,bfloat16,0,0.042936000227928164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,8,128,1,bfloat16,fp8,0,0.038343998789787295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,8,128,1,fp8,fp8,0,0.03877759873867035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,64,128,1,bfloat16,bfloat16,0,0.03877600133419037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,64,128,1,bfloat16,fp8,0,0.03073279857635498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,64,128,1,fp8,fp8,0,0.03132959902286529
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,1,128,1,bfloat16,bfloat16,0,0.02253919988870621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,1,128,1,bfloat16,fp8,0,0.023601600527763368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,1,128,1,fp8,fp8,0,0.023966400325298308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,2,128,1,bfloat16,bfloat16,0,0.022484800219535826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,2,128,1,bfloat16,fp8,0,0.023559999465942384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,2,128,1,fp8,fp8,0,0.023979200422763823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,4,128,1,bfloat16,bfloat16,0,0.02274879962205887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,4,128,1,bfloat16,fp8,0,0.023643200099468232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,4,128,1,fp8,fp8,0,0.02396959960460663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,8,128,1,bfloat16,bfloat16,0,0.02309280037879944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,8,128,1,bfloat16,fp8,0,0.02370239943265915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,64,8,128,1,fp8,fp8,0,0.024160000681877136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,64,8,128,1,bfloat16,bfloat16,0,0.14195040464401246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,1,128,1,bfloat16,bfloat16,0,1.5276816368103028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,1,128,1,bfloat16,fp8,0,1.5874896049499512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,1,128,1,fp8,fp8,0,1.5874624252319336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,2,128,1,bfloat16,bfloat16,0,1.5635647773742676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,64,8,128,1,bfloat16,fp8,0,0.07341279983520507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,64,1,128,1,bfloat16,fp8,0,0.24838719367980958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,2,128,1,bfloat16,fp8,0,1.5979583740234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,64,4,128,1,fp8,fp8,0,0.03776479959487915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,2,128,1,fp8,fp8,0,1.5883088111877441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,4,128,1,bfloat16,bfloat16,0,1.591324806213379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,4,128,1,bfloat16,fp8,0,1.6101087570190429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,4,128,1,fp8,fp8,0,1.610633659362793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,8,128,1,bfloat16,bfloat16,0,1.6961088180541992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,8,128,1,bfloat16,fp8,0,1.656697654724121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,64,8,128,1,fp8,fp8,0,1.6583759307861328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,64,128,1,bfloat16,bfloat16,0,1.5441472053527832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,1,128,1,bfloat16,bfloat16,0,0.7711120128631592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,64,128,1,bfloat16,fp8,0,1.1953984260559083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,64,128,1,fp8,fp8,0,1.1954511642456054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,1,128,1,bfloat16,fp8,0,0.8003168106079102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,1,128,1,fp8,fp8,0,0.8064543724060058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,2,128,1,bfloat16,bfloat16,0,0.7806608200073242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,2,128,1,bfloat16,fp8,0,0.8010463714599609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,4,128,1,bfloat16,bfloat16,0,0.8015376091003418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,2,128,1,fp8,fp8,0,0.8018575668334961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,4,128,1,bfloat16,fp8,0,0.8075152397155761
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,4,128,1,fp8,fp8,0,0.8033231735229492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,8,128,1,bfloat16,bfloat16,0,0.8456048011779785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,8,128,1,bfloat16,fp8,0,0.834665584564209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,64,128,1,bfloat16,bfloat16,0,0.7774400234222412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,64,8,128,1,fp8,fp8,0,0.8347855567932129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,64,128,1,bfloat16,fp8,0,0.6059328079223633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,1,128,1,bfloat16,bfloat16,0,0.3913280010223389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,64,128,1,fp8,fp8,0,0.6079696178436279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,1,128,1,bfloat16,fp8,0,0.4035439968109131
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,1,128,1,fp8,fp8,0,0.4056528091430664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,2,128,1,bfloat16,bfloat16,0,0.3963711977005005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,2,128,1,bfloat16,fp8,0,0.40656318664550783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,2,128,1,fp8,fp8,0,0.4048816204071045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,4,128,1,bfloat16,bfloat16,0,0.4066192150115967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,4,128,1,bfloat16,fp8,0,0.4132336139678955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,4,128,1,fp8,fp8,0,0.40944638252258303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,8,128,1,bfloat16,bfloat16,0,0.43071517944335935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,8,128,1,bfloat16,fp8,0,0.42189598083496094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,64,128,1,bfloat16,bfloat16,0,0.3963263988494873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,64,128,1,bfloat16,fp8,0,0.3102704048156738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,1,128,1,bfloat16,bfloat16,0,0.20211999416351317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,64,128,1,fp8,fp8,0,0.3116431951522827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,1,128,1,bfloat16,fp8,0,0.2085439920425415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,1,128,1,fp8,fp8,0,0.2076575994491577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,2,128,1,bfloat16,fp8,0,0.20782880783081054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,2,128,1,fp8,fp8,0,0.20764319896697997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,4,128,1,bfloat16,bfloat16,0,0.21152799129486083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,4,128,1,bfloat16,fp8,0,0.21164000034332275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,4,128,1,fp8,fp8,0,0.2106031894683838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,8,128,1,bfloat16,bfloat16,0,0.2209183931350708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,8,128,1,bfloat16,fp8,0,0.21659679412841798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,8,128,1,fp8,fp8,0,0.2180351972579956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,64,128,1,bfloat16,bfloat16,0,0.2061840057373047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,64,128,1,bfloat16,fp8,0,0.16171040534973144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,1,128,1,bfloat16,bfloat16,0,0.10728000402450562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,1,128,1,bfloat16,fp8,0,0.10962239503860474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,1,128,1,fp8,fp8,0,0.1098736047744751
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,2,128,1,bfloat16,bfloat16,0,0.10887839794158935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,2,128,1,fp8,fp8,0,0.11003520488739013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,4,128,1,bfloat16,bfloat16,0,0.11182880401611328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,4,128,1,bfloat16,fp8,0,0.11147680282592773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,4,128,1,fp8,fp8,0,0.11190240383148194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,8,128,1,bfloat16,bfloat16,0,0.11763199567794799
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,8,128,1,bfloat16,fp8,0,0.11469759941101074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,8,128,1,fp8,fp8,0,0.1143712043762207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,64,128,1,bfloat16,fp8,0,0.0886352002620697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,64,128,1,fp8,fp8,0,0.0881712019443512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,1,128,1,bfloat16,bfloat16,0,0.06087200045585632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,1,128,1,bfloat16,fp8,0,0.060303997993469236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,1,128,1,fp8,fp8,0,0.06119840145111084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,2,128,1,bfloat16,bfloat16,0,0.06095679998397827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,2,128,1,bfloat16,fp8,0,0.06118080019950867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,2,128,1,fp8,fp8,0,0.06092159748077393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,4,128,1,bfloat16,bfloat16,0,0.061926400661468504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,4,128,1,bfloat16,fp8,0,0.06172639727592468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,4,128,1,fp8,fp8,0,0.06210880279541016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,8,128,1,bfloat16,bfloat16,0,0.06487680077552796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,8,128,1,bfloat16,fp8,0,0.06333280205726624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,8,128,1,fp8,fp8,0,0.06305599808692933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,64,128,1,bfloat16,bfloat16,0,0.06100640296936035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,64,128,1,bfloat16,fp8,0,0.049451199173927304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,64,128,1,fp8,fp8,0,0.05012959837913513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,1,128,1,bfloat16,bfloat16,0,0.03213599920272827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,1,128,1,bfloat16,fp8,0,0.032630398869514465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,1,128,1,fp8,fp8,0,0.03261280059814453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,2,128,1,bfloat16,bfloat16,0,0.03356640040874481
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,2,128,1,bfloat16,fp8,0,0.03203360140323639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,2,128,1,fp8,fp8,0,0.03257279992103577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,4,128,1,bfloat16,bfloat16,0,0.03407039940357208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,4,128,1,bfloat16,fp8,0,0.03340159952640533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,4,128,1,fp8,fp8,0,0.03233759999275208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,8,128,1,bfloat16,bfloat16,0,0.03804000020027161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,8,128,1,bfloat16,fp8,0,0.03267680108547211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,64,8,128,1,fp8,fp8,0,0.421446418762207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,64,8,128,1,fp8,fp8,0,0.03353599905967712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,64,128,1,bfloat16,bfloat16,0,0.03576320111751556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,64,128,1,bfloat16,fp8,0,0.025572800636291505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,64,128,1,fp8,fp8,0,0.025675201416015626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,1,128,1,bfloat16,bfloat16,0,0.019156800210475923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,64,2,128,1,bfloat16,bfloat16,0,0.20370559692382811
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,1,128,1,bfloat16,fp8,0,0.020214399695396422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,1,128,1,fp8,fp8,0,0.020558400452136992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,2,128,1,bfloat16,fp8,0,0.020164799690246583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,4,128,1,bfloat16,bfloat16,0,0.019673599302768706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,4,128,1,bfloat16,fp8,0,0.020244799554347992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,4,128,1,fp8,fp8,0,0.020521600544452668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,8,128,1,bfloat16,bfloat16,0,0.019897599518299103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,8,128,1,bfloat16,fp8,0,0.02070239931344986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,8,128,1,fp8,fp8,0,0.020401600003242492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,64,128,1,bfloat16,bfloat16,0,0.017764799296855927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,64,128,1,bfloat16,fp8,0,0.01653279960155487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,64,128,1,fp8,fp8,0,0.016884799301624297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,1,128,1,bfloat16,bfloat16,0,0.014510400593280792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,64,128,1,fp8,fp8,0,0.16179840564727782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,1,128,1,bfloat16,fp8,0,0.015328000485897064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,1,128,1,fp8,fp8,0,0.014843200147151948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,2,128,1,bfloat16,bfloat16,0,0.01443679928779602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,2,128,1,bfloat16,fp8,0,0.01525920033454895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,2,128,1,fp8,fp8,0,0.014846399426460266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,64,2,128,1,bfloat16,fp8,0,0.10962079763412476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,4,128,1,bfloat16,bfloat16,0,0.014814400672912597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,4,128,1,bfloat16,fp8,0,0.01528320014476776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,4,128,1,fp8,fp8,0,0.014932799339294433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,8,128,1,bfloat16,bfloat16,0,0.014798399806022645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,8,128,1,bfloat16,fp8,0,0.015328000485897064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,64,8,128,1,fp8,fp8,0,0.01505119949579239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,64,64,128,1,bfloat16,bfloat16,0,0.11103999614715576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,1,128,1,bfloat16,bfloat16,0,0.7100207805633545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,1,128,1,bfloat16,fp8,0,0.7698880195617676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,1,128,1,fp8,fp8,0,0.7703199863433838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,2,128,1,bfloat16,bfloat16,0,0.7154160022735596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,2,128,1,bfloat16,fp8,0,0.7780543804168701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,2,128,1,fp8,fp8,0,0.7766160011291504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,4,128,1,bfloat16,bfloat16,0,0.7270127773284912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,4,128,1,bfloat16,fp8,0,0.7961567878723145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,4,128,1,fp8,fp8,0,0.7944767951965332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,2,128,1,bfloat16,bfloat16,0,0.019500799477100372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,64,2,128,1,fp8,fp8,0,0.02030239999294281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,8,128,1,bfloat16,bfloat16,0,0.7697904109954834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,8,128,1,bfloat16,fp8,0,0.832636833190918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,64,8,128,1,fp8,fp8,0,0.8325823783874512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,64,128,1,bfloat16,bfloat16,0,0.7596159934997558
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,1,128,1,bfloat16,bfloat16,0,0.3595168113708496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,64,128,1,bfloat16,fp8,0,0.6264832019805908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,1,128,1,bfloat16,fp8,0,0.39268479347229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,1,128,1,fp8,fp8,0,0.39236319065093994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,64,128,1,fp8,fp8,0,0.6263391971588135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,2,128,1,bfloat16,fp8,0,0.39605441093444826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,2,128,1,fp8,fp8,0,0.391484808921814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,4,128,1,bfloat16,bfloat16,0,0.3714447975158691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,4,128,1,bfloat16,fp8,0,0.40446081161499026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,4,128,1,fp8,fp8,0,0.40295839309692383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,8,128,1,bfloat16,bfloat16,0,0.3908688068389893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,8,128,1,bfloat16,fp8,0,0.4204063892364502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,1,128,1,bfloat16,bfloat16,0,0.18604320287704468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,8,128,1,fp8,fp8,0,0.4198336124420166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,64,128,1,bfloat16,fp8,0,0.31621599197387695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,64,128,1,fp8,fp8,0,0.3172352075576782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,1,128,1,bfloat16,fp8,0,0.1997167944908142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,1,128,1,fp8,fp8,0,0.20133121013641359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,2,128,1,bfloat16,bfloat16,0,0.18712799549102782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,2,128,1,bfloat16,fp8,0,0.20051200389862062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,2,128,1,fp8,fp8,0,0.20220959186553955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,4,128,1,bfloat16,bfloat16,0,0.19237120151519777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,4,128,1,bfloat16,fp8,0,0.20670719146728517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,4,128,1,fp8,fp8,0,0.20614879131317138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,8,128,1,bfloat16,fp8,0,0.2144831895828247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,8,128,1,fp8,fp8,0,0.21405439376831054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,64,128,1,bfloat16,bfloat16,0,0.20195360183715821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,64,128,1,bfloat16,fp8,0,0.16625920534133912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,1,128,1,bfloat16,bfloat16,0,0.1003648042678833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,1,128,1,bfloat16,fp8,0,0.10725760459899902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,1,128,1,fp8,fp8,0,0.1068303942680359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,2,128,1,bfloat16,bfloat16,0,0.10089279413223266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,2,128,1,bfloat16,fp8,0,0.1076464056968689
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,2,128,1,fp8,fp8,0,0.10830080509185791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,4,128,1,bfloat16,bfloat16,0,0.10318080186843873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,4,128,1,bfloat16,fp8,0,0.10907039642333985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,4,128,1,fp8,fp8,0,0.10936640501022339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,8,128,1,bfloat16,bfloat16,0,0.10816479921340942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,8,128,1,bfloat16,fp8,0,0.11352640390396118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,8,128,1,fp8,fp8,0,0.11387519836425782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,64,128,1,bfloat16,bfloat16,0,0.11122720241546631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,64,128,1,bfloat16,fp8,0,0.08765760064125061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,1,128,1,bfloat16,bfloat16,0,0.05667039752006531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,64,128,1,fp8,fp8,0,0.08743199706077576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,1,128,1,bfloat16,fp8,0,0.05869600176811218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,1,128,1,fp8,fp8,0,0.05785599946975708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,2,128,1,bfloat16,bfloat16,0,0.056145602464675905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,2,128,1,bfloat16,fp8,0,0.059543997049331665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,2,128,1,fp8,fp8,0,0.05866559743881226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,4,128,1,bfloat16,bfloat16,0,0.05740320086479187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,4,128,1,bfloat16,fp8,0,0.060228800773620604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,4,128,1,fp8,fp8,0,0.05904639959335327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,8,128,1,bfloat16,bfloat16,0,0.06045600175857544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,8,128,1,bfloat16,fp8,0,0.06173440217971802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,64,8,128,1,fp8,fp8,0,0.060734397172927855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,64,2,128,1,bfloat16,bfloat16,0,0.3630208015441895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,64,128,1,bfloat16,bfloat16,0,0.05904639959335327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,64,128,1,bfloat16,fp8,0,0.04705440104007721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,64,128,1,fp8,fp8,0,0.046803200244903566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,1,128,1,bfloat16,bfloat16,0,0.02863520085811615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,1,128,1,bfloat16,fp8,0,0.029390400648117064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,1,128,1,fp8,fp8,0,0.029942399263381957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,2,128,1,bfloat16,bfloat16,0,0.02905600070953369
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,2,128,1,bfloat16,fp8,0,0.029235199093818665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,2,128,1,fp8,fp8,0,0.029399999976158143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,4,128,1,bfloat16,bfloat16,0,0.030707201361656188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,4,128,1,bfloat16,fp8,0,0.029756799340248108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,4,128,1,fp8,fp8,0,0.02990880012512207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,8,128,1,bfloat16,fp8,0,0.03051519989967346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,8,128,1,bfloat16,bfloat16,0,0.03391200006008148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,64,8,128,1,fp8,fp8,0,0.02974880039691925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,64,128,1,bfloat16,bfloat16,0,0.03353599905967712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,64,128,1,bfloat16,fp8,0,0.021942399442195892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,64,128,1,fp8,fp8,0,0.021875199675559998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,1,128,1,bfloat16,bfloat16,0,0.01785759925842285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,1,128,1,bfloat16,fp8,0,0.01897120028734207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,1,128,1,fp8,fp8,0,0.01934719979763031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,2,128,1,bfloat16,bfloat16,0,0.017710399627685548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,2,128,1,bfloat16,fp8,0,0.018750399351119995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,2,128,1,fp8,fp8,0,0.018783999979496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,4,128,1,bfloat16,bfloat16,0,0.01828639954328537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,4,128,1,bfloat16,fp8,0,0.018603199720382692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,4,128,1,fp8,fp8,0,0.01896799951791763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,64,128,1,bfloat16,bfloat16,0,0.39014239311218263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,8,128,1,bfloat16,fp8,0,0.018775999546051025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,8,128,1,fp8,fp8,0,0.019079999625682832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,64,128,1,bfloat16,bfloat16,0,0.01600479930639267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,64,128,1,fp8,fp8,0,0.015038399398326874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,1,128,1,bfloat16,bfloat16,0,0.01311040073633194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,1,128,1,bfloat16,fp8,0,0.013831999897956849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,1,128,1,fp8,fp8,0,0.013724799454212188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,2,128,1,bfloat16,bfloat16,0,0.013142399489879608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,2,128,1,bfloat16,fp8,0,0.01348000019788742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,2,128,1,fp8,fp8,0,0.013742400705814362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,4,128,1,bfloat16,bfloat16,0,0.013027200102806091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,4,128,1,bfloat16,fp8,0,0.01374559998512268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,64,8,128,1,bfloat16,bfloat16,0,0.20069599151611328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,4,128,1,fp8,fp8,0,0.013665600121021271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,8,128,1,bfloat16,bfloat16,0,0.013553600013256072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,8,128,1,fp8,fp8,0,0.01369439959526062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,64,128,1,bfloat16,bfloat16,0,0.014446400105953217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,64,128,1,bfloat16,fp8,0,0.013900800049304963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,64,64,128,1,fp8,fp8,0,0.16516480445861817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,1,128,1,bfloat16,bfloat16,0,0.012572799623012543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,1,128,1,fp8,fp8,0,0.013438400626182557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,1,128,1,bfloat16,fp8,0,0.012995199859142303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,2,128,1,bfloat16,bfloat16,0,0.012540799379348756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,2,128,1,bfloat16,fp8,0,0.013574400544166565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,4,128,1,bfloat16,bfloat16,0,0.012676799297332763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,4,128,1,bfloat16,fp8,0,0.013577599823474885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,4,128,1,fp8,fp8,0,0.013342399895191193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,8,128,1,bfloat16,bfloat16,0,0.013041600584983826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,8,128,1,bfloat16,fp8,0,0.013212800025939941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,8,128,1,fp8,fp8,0,0.013393600285053254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,1,128,1,bfloat16,bfloat16,0,0.3647792100906372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,1,128,1,bfloat16,fp8,0,0.3936608076095581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,1,128,1,fp8,fp8,0,0.3947887897491455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,2,128,1,bfloat16,bfloat16,0,0.368120002746582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,2,128,1,bfloat16,fp8,0,0.39798080921173096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,2,128,1,fp8,fp8,0,0.3973104000091553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,64,8,128,1,bfloat16,bfloat16,0,0.018775999546051025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,4,128,1,bfloat16,bfloat16,0,0.3771712064743042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,64,128,1,bfloat16,fp8,0,0.014727999269962311
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,4,128,1,bfloat16,fp8,0,0.4036816120147705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,4,128,1,fp8,fp8,0,0.40699200630187987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,8,128,1,bfloat16,bfloat16,0,0.3952752113342285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,8,128,1,bfloat16,fp8,0,0.4191408157348633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,64,8,128,1,fp8,fp8,0,0.4217440128326416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,64,8,128,1,bfloat16,fp8,0,0.013651199638843536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,64,128,1,fp8,fp8,0,0.013916799426078796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,1,128,1,bfloat16,bfloat16,0,0.18827519416809083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,64,128,1,bfloat16,bfloat16,0,0.47944002151489257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,1,128,1,bfloat16,fp8,0,0.2020848035812378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,64,128,1,bfloat16,fp8,0,0.45719361305236816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,1,128,1,fp8,fp8,0,0.20191519260406493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,64,2,128,1,fp8,fp8,0,0.012856000661849975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,2,128,1,bfloat16,bfloat16,0,0.18882240056991578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,64,128,1,fp8,fp8,0,0.45764479637145994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,2,128,1,bfloat16,fp8,0,0.2042383909225464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,2,128,1,fp8,fp8,0,0.20402240753173828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,4,128,1,bfloat16,bfloat16,0,0.19423680305480956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,4,128,1,bfloat16,fp8,0,0.2057408094406128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,4,128,1,fp8,fp8,0,0.20762081146240235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,8,128,1,bfloat16,fp8,0,0.21365759372711182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,8,128,1,fp8,fp8,0,0.21609120368957518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,64,128,1,bfloat16,fp8,0,0.23570239543914795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,64,128,1,bfloat16,bfloat16,0,0.24820799827575685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,64,128,1,fp8,fp8,0,0.23649280071258544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,1,128,1,bfloat16,bfloat16,0,0.10126719474792481
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,1,128,1,bfloat16,fp8,0,0.1081536054611206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,1,128,1,fp8,fp8,0,0.10710400342941284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,2,128,1,bfloat16,bfloat16,0,0.10052160024642945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,2,128,1,fp8,fp8,0,0.1089087963104248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,2,128,1,bfloat16,fp8,0,0.10831999778747559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,4,128,1,bfloat16,bfloat16,0,0.10406559705734253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,4,128,1,fp8,fp8,0,0.10977439880371094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,8,128,1,bfloat16,bfloat16,0,0.10879199504852295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,8,128,1,bfloat16,fp8,0,0.11391040086746215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,8,128,1,fp8,fp8,0,0.11457760334014892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,64,128,1,bfloat16,fp8,0,0.12227360010147095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,64,128,1,bfloat16,bfloat16,0,0.13148319721221924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,1,128,1,bfloat16,bfloat16,0,0.05758559703826904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,1,128,1,bfloat16,fp8,0,0.05926719903945923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,1,128,1,fp8,fp8,0,0.05873280167579651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,2,128,1,bfloat16,bfloat16,0,0.058246397972106935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,2,128,1,bfloat16,fp8,0,0.0593280017375946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,2,128,1,fp8,fp8,0,0.05949599742889404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,4,128,1,bfloat16,bfloat16,0,0.0586143970489502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,4,128,1,fp8,fp8,0,0.0593887984752655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,8,128,1,bfloat16,bfloat16,0,0.06155840158462524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,8,128,1,bfloat16,fp8,0,0.0613647997379303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,8,128,1,fp8,fp8,0,0.06170240044593811
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,64,128,1,bfloat16,fp8,0,0.0640496015548706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,64,128,1,fp8,fp8,0,0.06373119950294495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,1,128,1,bfloat16,bfloat16,0,0.0279695987701416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,1,128,1,bfloat16,fp8,0,0.02937600016593933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,1,128,1,fp8,fp8,0,0.0292959988117218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,2,128,1,bfloat16,bfloat16,0,0.02908799946308136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,2,128,1,bfloat16,fp8,0,0.029212799668312073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,2,128,1,fp8,fp8,0,0.02929919958114624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,4,128,1,bfloat16,bfloat16,0,0.03062559962272644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,4,128,1,bfloat16,fp8,0,0.02954080104827881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,4,128,1,fp8,fp8,0,0.02964160144329071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,8,128,1,bfloat16,bfloat16,0,0.03443039953708649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,8,128,1,bfloat16,fp8,0,0.029860800504684447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,8,128,1,fp8,fp8,0,0.029918399453163148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,64,128,1,bfloat16,bfloat16,0,0.04097760021686554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,64,128,1,bfloat16,fp8,0,0.030780801177024843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,64,128,1,fp8,fp8,0,0.030888000130653383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,1,128,1,bfloat16,bfloat16,0,0.017895999550819396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,1,128,1,bfloat16,fp8,0,0.019011199474334717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,1,128,1,fp8,fp8,0,0.018915200233459474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,2,128,1,bfloat16,bfloat16,0,0.017764799296855927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,2,128,1,bfloat16,fp8,0,0.01886560022830963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,2,128,1,fp8,fp8,0,0.018904000520706177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,4,128,1,bfloat16,bfloat16,0,0.018104000389575957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,4,128,1,bfloat16,fp8,0,0.018982400000095368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,4,128,1,fp8,fp8,0,0.01905599981546402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,8,128,1,bfloat16,bfloat16,0,0.018254399299621582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,8,128,1,bfloat16,fp8,0,0.01919520050287247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,64,8,128,1,fp8,fp8,0,0.019193600118160247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,64,128,1,bfloat16,bfloat16,0,0.019617600739002226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,64,128,1,fp8,fp8,0,0.018848000466823576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,64,4,128,1,bfloat16,fp8,0,0.11095999479293824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,1,128,1,bfloat16,bfloat16,0,0.012955200672149659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,1,128,1,bfloat16,fp8,0,0.013513599336147309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,1,128,1,fp8,fp8,0,0.01377120018005371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,2,128,1,bfloat16,bfloat16,0,0.012856000661849975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,64,128,1,fp8,fp8,0,0.12278879880905151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,2,128,1,bfloat16,fp8,0,0.013732799887657165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,2,128,1,fp8,fp8,0,0.01345440000295639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,4,128,1,bfloat16,bfloat16,0,0.013129599392414093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,4,128,1,fp8,fp8,0,0.013555200397968292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,8,128,1,bfloat16,bfloat16,0,0.013419200479984284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,8,128,1,bfloat16,fp8,0,0.013710400462150574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,64,4,128,1,bfloat16,fp8,0,0.06001920104026794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,8,128,1,fp8,fp8,0,0.013873599469661713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,64,128,1,bfloat16,bfloat16,0,0.013966399431228637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,64,128,1,bfloat16,fp8,0,0.01395999938249588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,64,128,1,fp8,fp8,0,0.013887999951839447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,1,128,1,bfloat16,fp8,0,0.013236799836158752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,64,64,128,1,bfloat16,bfloat16,0,0.0713599979877472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,1,128,1,fp8,fp8,0,0.013182400166988373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,2,128,1,bfloat16,bfloat16,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,2,128,1,bfloat16,fp8,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,2,128,1,fp8,fp8,0,0.013079999387264252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,4,128,1,bfloat16,bfloat16,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,4,128,1,bfloat16,fp8,0,0.013406400382518769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,4,128,1,fp8,fp8,0,0.01313920021057129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,8,128,1,bfloat16,bfloat16,0,0.01271039992570877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,8,128,1,bfloat16,fp8,0,0.013417600095272065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,8,128,1,fp8,fp8,0,0.013419200479984284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,64,128,1,bfloat16,bfloat16,0,0.013566400110721587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,64,128,1,bfloat16,fp8,0,0.013622400164604188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,1,128,1,bfloat16,bfloat16,0,0.012280000001192093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,1,128,1,bfloat16,fp8,0,0.013407999277114868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,1,128,1,fp8,fp8,0,0.013049599528312684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,2,128,1,bfloat16,bfloat16,0,0.012443199753761292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,2,128,1,bfloat16,fp8,0,0.013036799430847169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,2,128,1,fp8,fp8,0,0.013145600259304047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,4,128,1,bfloat16,bfloat16,0,0.012411200255155564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,4,128,1,bfloat16,fp8,0,0.01313759982585907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,4,128,1,fp8,fp8,0,0.013235199451446533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,8,128,1,bfloat16,bfloat16,0,0.012671999633312225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,8,128,1,bfloat16,fp8,0,0.013132800161838532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,8,128,1,fp8,fp8,0,0.012940800189971924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,1,128,1,bfloat16,bfloat16,0,0.18612799644470215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,1,128,1,bfloat16,fp8,0,0.20308320522308348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,1,128,1,fp8,fp8,0,0.20231521129608154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,2,128,1,bfloat16,bfloat16,0,0.18959840536117553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,2,128,1,bfloat16,fp8,0,0.20334720611572266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,2,128,1,fp8,fp8,0,0.20419518947601317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,64,8,128,1,bfloat16,bfloat16,0,0.20420639514923095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,4,128,1,bfloat16,bfloat16,0,0.19360320568084716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,4,128,1,bfloat16,fp8,0,0.20686719417572022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,4,128,1,fp8,fp8,0,0.20605120658874512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,64,128,1,bfloat16,fp8,0,0.018878400325775146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,8,128,1,bfloat16,bfloat16,0,0.2048352003097534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,8,128,1,bfloat16,fp8,0,0.2144495964050293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,64,8,128,1,fp8,fp8,0,0.21447360515594482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,64,128,1,bfloat16,bfloat16,0,0.36323840618133546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,1,128,1,bfloat16,bfloat16,0,0.10052319765090942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,64,128,1,bfloat16,fp8,0,0.3669935941696167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,64,128,1,fp8,fp8,0,0.3672384023666382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,1,128,1,bfloat16,fp8,0,0.10900479555130005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,1,128,1,fp8,fp8,0,0.10896639823913574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,2,128,1,bfloat16,bfloat16,0,0.1016160011291504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,2,128,1,bfloat16,fp8,0,0.10940320491790771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,2,128,1,fp8,fp8,0,0.10982240438461303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,64,1,128,1,bfloat16,bfloat16,0,0.012563200294971466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,4,128,1,bfloat16,bfloat16,0,0.10448160171508789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,4,128,1,bfloat16,fp8,0,0.1109231948852539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,4,128,1,fp8,fp8,0,0.11013920307159424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,8,128,1,fp8,fp8,0,0.113864004611969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,64,128,1,bfloat16,bfloat16,0,0.18879040479660034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,64,128,1,bfloat16,fp8,0,0.19240000247955322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,1,128,1,bfloat16,bfloat16,0,0.05785120129585266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,64,128,1,fp8,fp8,0,0.1934239983558655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,64,64,128,1,fp8,fp8,0,0.013643200695514678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,1,128,1,bfloat16,fp8,0,0.05942559838294983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,1,128,1,fp8,fp8,0,0.0596671998500824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,2,128,1,bfloat16,fp8,0,0.05952640175819397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,2,128,1,fp8,fp8,0,0.06006720066070557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,4,128,1,bfloat16,bfloat16,0,0.059683197736740114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,4,128,1,bfloat16,fp8,0,0.060759997367858885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,4,128,1,fp8,fp8,0,0.06079040169715881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,8,128,1,bfloat16,bfloat16,0,0.06196640133857727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,8,128,1,bfloat16,fp8,0,0.06267039775848389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,8,128,1,fp8,fp8,0,0.06269599795341492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,64,128,1,bfloat16,bfloat16,0,0.10165599584579468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,64,128,1,bfloat16,fp8,0,0.10012320280075074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,64,128,1,fp8,fp8,0,0.10051200389862061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,1,128,1,bfloat16,bfloat16,0,0.02866080105304718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,1,128,1,bfloat16,fp8,0,0.029702401161193846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,1,128,1,fp8,fp8,0,0.029524800181388856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,2,128,1,bfloat16,bfloat16,0,0.02908959984779358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,2,128,1,bfloat16,fp8,0,0.029633599519729614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,2,128,1,fp8,fp8,0,0.029444798827171326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,4,128,1,bfloat16,bfloat16,0,0.03107840120792389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,4,128,1,bfloat16,fp8,0,0.02972640097141266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,4,128,1,fp8,fp8,0,0.029812800884246825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,8,128,1,bfloat16,bfloat16,0,0.0340175986289978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,8,128,1,bfloat16,fp8,0,0.030432000756263733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,64,8,128,1,fp8,fp8,0,0.029791998863220214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,64,128,1,bfloat16,bfloat16,0,0.05602399706840515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,64,128,1,bfloat16,fp8,0,0.04975999891757965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,64,128,1,fp8,fp8,0,0.04902400076389313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,1,128,1,bfloat16,bfloat16,0,0.01802240014076233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,1,128,1,bfloat16,fp8,0,0.018943999707698823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,1,128,1,fp8,fp8,0,0.018889600038528444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,2,128,1,bfloat16,bfloat16,0,0.01812479943037033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,2,128,1,bfloat16,fp8,0,0.018862399458885192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,2,128,1,fp8,fp8,0,0.01883520036935806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,4,128,1,bfloat16,fp8,0,0.0189423993229866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,4,128,1,fp8,fp8,0,0.019006399810314177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,8,128,1,bfloat16,bfloat16,0,0.018196800351142885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,8,128,1,bfloat16,fp8,0,0.019019199907779692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,8,128,1,fp8,fp8,0,0.018985599279403687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,64,128,1,bfloat16,bfloat16,0,0.02638559937477112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,64,128,1,bfloat16,fp8,0,0.027854400873184203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,64,128,1,fp8,fp8,0,0.02787039875984192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,64,4,128,1,bfloat16,fp8,0,0.01374559998512268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,1,128,1,bfloat16,fp8,0,0.013819199800491334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,1,128,1,fp8,fp8,0,0.014215999841690063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,2,128,1,bfloat16,bfloat16,0,0.012881599366664886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,2,128,1,bfloat16,fp8,0,0.01393440067768097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,2,128,1,fp8,fp8,0,0.01371839940547943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,4,128,1,bfloat16,bfloat16,0,0.013172799348831176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,4,128,1,bfloat16,fp8,0,0.013582399487495423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,4,128,1,fp8,fp8,0,0.013755199313163758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,8,128,1,bfloat16,bfloat16,0,0.013300800323486328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,8,128,1,bfloat16,fp8,0,0.014025600254535675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,8,128,1,bfloat16,fp8,0,0.11461280584335327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,8,128,1,fp8,fp8,0,0.013784000277519226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,64,128,1,bfloat16,bfloat16,0,0.01764000058174133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,64,128,1,bfloat16,fp8,0,0.01825760006904602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,64,128,1,fp8,fp8,0,0.017979200184345245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,1,128,1,bfloat16,bfloat16,0,0.012417600303888322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,1,128,1,fp8,fp8,0,0.01313440054655075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,2,128,1,bfloat16,bfloat16,0,0.012537600100040435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,1,128,1,bfloat16,fp8,0,0.013414399325847625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,2,128,1,bfloat16,fp8,0,0.013307200372219085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,2,128,1,fp8,fp8,0,0.01318880021572113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,64,2,128,1,bfloat16,bfloat16,0,0.05839840173721313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,4,128,1,bfloat16,bfloat16,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,4,128,1,bfloat16,fp8,0,0.013107199966907502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,4,128,1,fp8,fp8,0,0.013259199261665345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,8,128,1,bfloat16,bfloat16,0,0.012865599989891053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,8,128,1,bfloat16,fp8,0,0.013395200669765472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,64,8,128,1,fp8,fp8,0,0.013252800703048706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,64,128,1,bfloat16,fp8,0,0.013678400218486786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,64,128,1,fp8,fp8,0,0.013606399297714233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,1,128,1,bfloat16,bfloat16,0,0.01234399974346161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,1,128,1,bfloat16,fp8,0,0.01327199935913086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,1,128,1,fp8,fp8,0,0.013260799646377563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,2,128,1,bfloat16,bfloat16,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,2,128,1,bfloat16,fp8,0,0.013199999928474426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,2,128,1,fp8,fp8,0,0.012918399274349212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,4,128,1,bfloat16,bfloat16,0,0.01220960021018982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,4,128,1,bfloat16,fp8,0,0.013199999928474426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,4,128,1,fp8,fp8,0,0.012945599853992462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,8,128,1,bfloat16,bfloat16,0,0.012476799637079239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,8,128,1,bfloat16,fp8,0,0.013382400572299957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,8,128,1,fp8,fp8,0,0.013019199669361114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,64,128,1,bfloat16,bfloat16,0,0.013486400246620178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,64,128,1,fp8,fp8,0,0.013550400733947754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,1,128,1,bfloat16,bfloat16,0,0.012198399752378464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,1,128,1,bfloat16,fp8,0,0.012875199317932129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,1,128,1,fp8,fp8,0,0.012801599502563477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,2,128,1,bfloat16,bfloat16,0,0.012148799747228623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,2,128,1,bfloat16,fp8,0,0.012839999794960023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,2,128,1,fp8,fp8,0,0.012833599746227265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,4,128,1,bfloat16,bfloat16,0,0.012247999757528305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,4,128,1,bfloat16,fp8,0,0.012809599936008453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,4,128,1,fp8,fp8,0,0.012862400710582733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,8,128,1,bfloat16,bfloat16,0,0.012206400185823441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,8,128,1,bfloat16,fp8,0,0.01305440068244934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,8,128,1,fp8,fp8,0,0.012908799946308136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,1,128,1,bfloat16,bfloat16,0,0.10085439682006836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,1,128,1,bfloat16,fp8,0,0.11063519716262818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,1,128,1,fp8,fp8,0,0.11008800268173217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,64,4,128,1,bfloat16,bfloat16,0,0.0182559996843338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,2,128,1,bfloat16,bfloat16,0,0.10233759880065918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,2,128,1,bfloat16,fp8,0,0.11080479621887207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,2,128,1,fp8,fp8,0,0.11119840145111085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,4,128,1,bfloat16,bfloat16,0,0.10572479963302613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,64,1,128,1,bfloat16,bfloat16,0,0.013031999766826629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,4,128,1,bfloat16,fp8,0,0.11197600364685059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,4,128,1,fp8,fp8,0,0.11228159666061402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,8,128,1,bfloat16,bfloat16,0,0.11183359622955322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,8,128,1,bfloat16,fp8,0,0.11556799411773681
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,64,8,128,1,fp8,fp8,0,0.11545920372009277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,64,8,128,1,bfloat16,bfloat16,0,0.10999679565429688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,64,128,1,bfloat16,bfloat16,0,0.3081775903701782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,64,128,1,bfloat16,fp8,0,0.3370752096176147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,1,128,1,bfloat16,bfloat16,0,0.05809119939804077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,1,128,1,bfloat16,fp8,0,0.06121439933776855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,1,128,1,fp8,fp8,0,0.06068959832191467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,64,128,1,fp8,fp8,0,0.33653600215911866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,2,128,1,bfloat16,bfloat16,0,0.05871840119361878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,2,128,1,bfloat16,fp8,0,0.06145600080490112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,4,128,1,bfloat16,bfloat16,0,0.060068798065185544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,4,128,1,fp8,fp8,0,0.06213120222091675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,8,128,1,bfloat16,bfloat16,0,0.06268799901008607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,8,128,1,bfloat16,fp8,0,0.06335520148277282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,8,128,1,fp8,fp8,0,0.06336159706115722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,64,64,128,1,bfloat16,bfloat16,0,0.013625599443912506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,64,128,1,bfloat16,bfloat16,0,0.16006720066070557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,64,128,1,bfloat16,fp8,0,0.1736863970756531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,1,128,1,bfloat16,bfloat16,0,0.02905279994010925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,1,128,1,bfloat16,fp8,0,0.030214399099349976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,1,128,1,fp8,fp8,0,0.030048000812530517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,2,128,1,bfloat16,bfloat16,0,0.029203200340270997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,2,128,1,fp8,fp8,0,0.02999520003795624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,4,128,1,bfloat16,bfloat16,0,0.030667200684547424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,4,128,1,bfloat16,fp8,0,0.03027839958667755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,64,64,128,1,bfloat16,fp8,0,0.013654400408267976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,4,128,1,fp8,fp8,0,0.030267199873924254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,8,128,1,bfloat16,bfloat16,0,0.03452320098876953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,8,128,1,bfloat16,fp8,0,0.03075680136680603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,8,128,1,fp8,fp8,0,0.03126879930496216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,64,128,1,bfloat16,bfloat16,0,0.08716319799423218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,64,128,1,bfloat16,fp8,0,0.08617920279502869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,1,128,1,bfloat16,bfloat16,0,0.018190400302410127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,1,128,1,bfloat16,fp8,0,0.0196383997797966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,1,128,1,fp8,fp8,0,0.01937599927186966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,2,128,1,bfloat16,bfloat16,0,0.018198400735855103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,2,128,1,bfloat16,fp8,0,0.019254399836063384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,2,128,1,fp8,fp8,0,0.019492800533771514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,4,128,1,bfloat16,fp8,0,0.019472000002861024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,4,128,1,fp8,fp8,0,0.019436800479888917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,8,128,1,bfloat16,bfloat16,0,0.01844799965620041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,8,128,1,bfloat16,fp8,0,0.019467200338840484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,8,128,1,fp8,fp8,0,0.019470399618148802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,64,128,1,bfloat16,bfloat16,0,0.04140160083770752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,64,128,1,bfloat16,fp8,0,0.04637120068073273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,64,128,1,fp8,fp8,0,0.04648799896240234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,1,128,1,bfloat16,bfloat16,0,0.013195200264453888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,1,128,1,bfloat16,fp8,0,0.014097599685192109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,1,128,1,fp8,fp8,0,0.013952000439167023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,2,128,1,bfloat16,bfloat16,0,0.013169600069522858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,2,128,1,bfloat16,fp8,0,0.013894400000572205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,2,128,1,fp8,fp8,0,0.013766400516033173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,4,128,1,bfloat16,bfloat16,0,0.013208000361919403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,4,128,1,bfloat16,fp8,0,0.013761599361896516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,2,128,1,fp8,fp8,0,0.06154879927635193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,64,4,128,1,bfloat16,fp8,0,0.061924797296524045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,8,128,1,bfloat16,bfloat16,0,0.013625599443912506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,8,128,1,bfloat16,fp8,0,0.014025600254535675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,8,128,1,fp8,fp8,0,0.013966399431228637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,64,128,1,bfloat16,bfloat16,0,0.025148800015449523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,64,128,1,bfloat16,fp8,0,0.02751680016517639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,64,128,1,fp8,fp8,0,0.027113598585128785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,64,128,1,fp8,fp8,0,0.175108802318573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,1,128,1,fp8,fp8,0,0.013257600367069244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,2,128,1,bfloat16,bfloat16,0,0.012641599774360657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,2,128,1,bfloat16,fp8,0,0.01318880021572113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,2,128,1,fp8,fp8,0,0.01329279989004135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,4,128,1,bfloat16,bfloat16,0,0.012670400738716125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,4,128,1,bfloat16,fp8,0,0.013115200400352477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,64,128,1,fp8,fp8,0,0.08643040060997009
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,4,128,1,fp8,fp8,0,0.013492800295352936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,8,128,1,bfloat16,bfloat16,0,0.013016000390052795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,8,128,1,bfloat16,fp8,0,0.013364799320697784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,8,128,1,fp8,fp8,0,0.013516800105571746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,64,128,1,bfloat16,bfloat16,0,0.017476800084114074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,64,128,1,bfloat16,fp8,0,0.01789119988679886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,64,128,1,fp8,fp8,0,0.018036800622940063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,1,128,1,bfloat16,bfloat16,0,0.012595200538635254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,64,4,128,1,bfloat16,bfloat16,0,0.018105599284172057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,1,128,1,bfloat16,fp8,0,0.013040000200271606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,1,128,1,fp8,fp8,0,0.013512000441551208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,2,128,1,bfloat16,bfloat16,0,0.012627199292182922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,2,128,1,fp8,fp8,0,0.013075199723243714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,4,128,1,bfloat16,bfloat16,0,0.012830400466918945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,4,128,1,bfloat16,fp8,0,0.0134320005774498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,4,128,1,fp8,fp8,0,0.013099199533462525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,8,128,1,bfloat16,bfloat16,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,8,128,1,bfloat16,fp8,0,0.013247999548912048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,8,128,1,fp8,fp8,0,0.013079999387264252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,64,128,1,bfloat16,bfloat16,0,0.013708800077438354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,64,128,1,bfloat16,fp8,0,0.013675199449062347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,64,128,1,fp8,fp8,0,0.01366720050573349
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,1,128,1,bfloat16,bfloat16,0,0.012201599776744843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,1,128,1,bfloat16,fp8,0,0.012812800705432892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,1,128,1,fp8,fp8,0,0.01284639984369278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,2,128,1,bfloat16,bfloat16,0,0.012428800016641617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,2,128,1,bfloat16,fp8,0,0.012835200130939483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,2,128,1,fp8,fp8,0,0.012830400466918945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,4,128,1,bfloat16,bfloat16,0,0.012363199889659882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,4,128,1,bfloat16,fp8,0,0.012835200130939483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,4,128,1,fp8,fp8,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,8,128,1,bfloat16,bfloat16,0,0.012508800625801087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,8,128,1,bfloat16,fp8,0,0.012718400359153748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,64,8,128,1,fp8,fp8,0,0.012884800136089326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,64,128,1,bfloat16,bfloat16,0,0.013299199938774108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,64,128,1,bfloat16,fp8,0,0.013468800485134125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,64,128,1,fp8,fp8,0,0.013369600474834441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,1,128,1,bfloat16,bfloat16,0,0.012081599980592727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,1,128,1,bfloat16,fp8,0,0.01255200058221817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,1,128,1,fp8,fp8,0,0.012833599746227265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,2,128,1,bfloat16,bfloat16,0,0.012019199877977371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,2,128,1,bfloat16,fp8,0,0.012777599692344665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,2,128,1,fp8,fp8,0,0.012683199346065521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,4,128,1,bfloat16,bfloat16,0,0.01199520006775856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,4,128,1,bfloat16,fp8,0,0.012667199969291687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,4,128,1,fp8,fp8,0,0.012670400738716125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,8,128,1,bfloat16,bfloat16,0,0.012196800112724305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,8,128,1,bfloat16,fp8,0,0.012798400223255157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,64,8,128,1,fp8,fp8,0,0.012806400656700134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,64,4,128,1,fp8,fp8,0,0.013822400569915771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,1,128,1,bfloat16,bfloat16,0,0.021236799657344818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,1,128,1,bfloat16,fp8,0,0.022675199806690215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,1,128,1,fp8,fp8,0,0.022614400088787078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,2,128,1,bfloat16,bfloat16,0,0.028438401222229005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,2,128,1,bfloat16,fp8,0,0.03120959997177124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,2,128,1,fp8,fp8,0,0.031281599402427675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,4,128,1,bfloat16,bfloat16,0,0.043219199776649474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,4,128,1,bfloat16,fp8,0,0.048790401220321654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,4,128,1,fp8,fp8,0,0.04889439940452576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,1,128,1,bfloat16,bfloat16,0,0.012649600207805634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,8,128,1,bfloat16,fp8,0,0.08407520055770874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,64,1,128,1,bfloat16,fp8,0,0.01311040073633194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,8,128,1,fp8,fp8,0,0.08436480164527893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,1,128,1,bfloat16,bfloat16,0,0.015292799472808838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,64,128,1,bfloat16,bfloat16,0,0.23957600593566894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,1,128,1,bfloat16,fp8,0,0.015593600273132325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,64,128,1,bfloat16,fp8,0,0.29534239768981935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,2,128,1,bfloat16,bfloat16,0,0.01833920031785965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,64,128,1,fp8,fp8,0,0.29576959609985354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,1,128,1,fp8,fp8,0,0.01557600051164627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,2,128,1,fp8,fp8,0,0.01993599981069565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,4,128,1,bfloat16,bfloat16,0,0.02588160037994385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,4,128,1,bfloat16,fp8,0,0.029080000519752503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,4,128,1,fp8,fp8,0,0.028937599062919615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,8,128,1,bfloat16,bfloat16,0,0.04031200110912323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,8,128,1,bfloat16,fp8,0,0.04674080014228821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,8,128,1,fp8,fp8,0,0.046563199162483214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,64,128,1,bfloat16,bfloat16,0,0.1270784020423889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,64,128,1,bfloat16,fp8,0,0.15538400411605835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,1,128,1,bfloat16,bfloat16,0,0.013350400328636169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,64,128,1,fp8,fp8,0,0.15462559461593628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,1,128,1,bfloat16,fp8,0,0.014235199987888336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,1,128,1,fp8,fp8,0,0.014150400459766389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,2,128,1,bfloat16,bfloat16,0,0.013686400651931763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,2,128,1,bfloat16,fp8,0,0.014387199282646179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,2,128,1,fp8,fp8,0,0.014440000057220459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,64,2,128,1,bfloat16,fp8,0,0.01308320015668869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,4,128,1,bfloat16,bfloat16,0,0.01753759980201721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,4,128,1,bfloat16,fp8,0,0.01886560022830963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,4,128,1,fp8,fp8,0,0.018795199692249298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,8,128,1,bfloat16,bfloat16,0,0.02481919974088669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,8,128,1,bfloat16,fp8,0,0.02775520086288452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,64,8,128,1,fp8,fp8,0,0.027687999606132507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,64,128,1,bfloat16,bfloat16,0,0.06994720101356507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,1,128,1,bfloat16,bfloat16,0,0.012624000012874604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,64,128,1,bfloat16,fp8,0,0.0816976010799408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,64,128,1,fp8,fp8,0,0.08224959969520569
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,1,128,1,fp8,fp8,0,0.013575999438762665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,2,128,1,bfloat16,bfloat16,0,0.012740799784660339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,2,128,1,bfloat16,fp8,0,0.013657599687576294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,2,128,1,fp8,fp8,0,0.013574400544166565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,4,128,1,bfloat16,bfloat16,0,0.012809599936008453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,4,128,1,bfloat16,fp8,0,0.01366720050573349
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,4,128,1,fp8,fp8,0,0.013604800403118133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,8,128,1,bfloat16,bfloat16,0,0.016649599373340606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,8,128,1,bfloat16,fp8,0,0.018030400574207305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,8,128,1,fp8,fp8,0,0.017918400466442108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,64,128,1,bfloat16,bfloat16,0,0.039585599303245546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,64,128,1,bfloat16,fp8,0,0.045193600654602054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,64,128,1,fp8,fp8,0,0.045351999998092654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,1,128,1,bfloat16,bfloat16,0,0.011903999745845795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,1,128,1,bfloat16,fp8,0,0.012747199833393097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,1,128,1,fp8,fp8,0,0.012848000228404998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,2,128,1,bfloat16,bfloat16,0,0.011972799897193909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,2,128,1,bfloat16,fp8,0,0.012756800651550293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,2,128,1,fp8,fp8,0,0.012868799269199371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,4,128,1,bfloat16,bfloat16,0,0.01204800009727478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,4,128,1,bfloat16,fp8,0,0.012999999523162841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,4,128,1,fp8,fp8,0,0.012782399356365205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,8,128,1,bfloat16,bfloat16,0,0.012097600102424621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,8,128,1,bfloat16,fp8,0,0.012947200238704682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,64,8,128,1,fp8,fp8,0,0.012942400574684144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,64,128,1,bfloat16,bfloat16,0,0.024691200256347655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,64,128,1,bfloat16,fp8,0,0.026897600293159483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,64,128,1,fp8,fp8,0,0.026638400554656983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,1,128,1,bfloat16,bfloat16,0,0.01175680011510849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,1,128,1,bfloat16,fp8,0,0.012428800016641617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,2,128,1,bfloat16,bfloat16,0,0.012043199688196182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,2,128,1,bfloat16,fp8,0,0.012481600046157837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,2,128,1,fp8,fp8,0,0.01279519945383072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,4,128,1,bfloat16,bfloat16,0,0.011761599779129028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,4,128,1,bfloat16,fp8,0,0.012886400520801543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,4,128,1,fp8,fp8,0,0.012617599964141846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,8,128,1,bfloat16,bfloat16,0,0.012241599708795547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,8,128,1,bfloat16,fp8,0,0.012694400548934937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,8,128,1,fp8,fp8,0,0.012848000228404998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,64,128,1,bfloat16,bfloat16,0,0.017020800709724428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,64,128,1,bfloat16,fp8,0,0.01777759939432144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,64,128,1,fp8,fp8,0,0.017529599368572235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,64,8,128,1,bfloat16,bfloat16,0,0.0714464008808136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,1,128,1,bfloat16,fp8,0,0.012214399874210358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,2,128,1,bfloat16,bfloat16,0,0.011548800021409988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,1,128,1,fp8,fp8,0,0.012587200105190276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,2,128,1,bfloat16,fp8,0,0.012176000326871873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,2,128,1,fp8,fp8,0,0.012521600723266602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,4,128,1,bfloat16,bfloat16,0,0.01167839989066124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,4,128,1,bfloat16,fp8,0,0.012593600153923034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,4,128,1,fp8,fp8,0,0.012355200201272964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,8,128,1,bfloat16,bfloat16,0,0.011974400281906128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,8,128,1,bfloat16,fp8,0,0.012479999661445617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,64,2,128,1,bfloat16,fp8,0,0.019840000569820403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,64,128,1,bfloat16,bfloat16,0,0.013116799294948578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,64,128,1,bfloat16,fp8,0,0.013492800295352936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,64,128,1,fp8,fp8,0,0.013235199451446533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,1,128,1,bfloat16,bfloat16,0,0.011631999909877778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,1,128,1,bfloat16,fp8,0,0.012240000069141388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,1,128,1,fp8,fp8,0,0.01239520013332367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,2,128,1,bfloat16,bfloat16,0,0.011436799913644791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,2,128,1,bfloat16,fp8,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,2,128,1,fp8,fp8,0,0.01215519979596138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,4,128,1,bfloat16,bfloat16,0,0.011643200367689132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,4,128,1,bfloat16,fp8,0,0.012291199713945388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,4,128,1,fp8,fp8,0,0.012303999811410903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,8,128,1,bfloat16,bfloat16,0,0.011608000099658965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,8,128,1,bfloat16,fp8,0,0.012452799826860428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,64,8,128,1,fp8,fp8,0,0.012444800138473511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,64,128,1,bfloat16,bfloat16,0,0.013251200318336487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,64,128,1,bfloat16,fp8,0,0.013155199587345123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,64,128,1,fp8,fp8,0,0.013313600420951843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,1,128,1,bfloat16,bfloat16,0,0.011444800347089768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,1,128,1,bfloat16,fp8,0,0.012272000312805176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,1,128,1,fp8,fp8,0,0.012145599722862244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,64,1,128,1,bfloat16,fp8,0,0.013478399813175201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,2,128,1,bfloat16,fp8,0,0.012142399698495865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,2,128,1,fp8,fp8,0,0.012220799922943115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,4,128,1,bfloat16,bfloat16,0,0.011432000249624253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,4,128,1,bfloat16,fp8,0,0.012243200093507767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,4,128,1,fp8,fp8,0,0.012135999649763108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,8,128,1,bfloat16,bfloat16,0,0.011457599699497223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,8,128,1,bfloat16,fp8,0,0.012198399752378464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,8,128,1,fp8,fp8,0,0.012324800342321396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,64,1,128,1,fp8,fp8,0,0.012577599287033081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,1,128,1,bfloat16,bfloat16,0,0.011686400324106217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,64,2,128,1,bfloat16,fp8,0,0.03022400140762329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,64,8,128,1,fp8,fp8,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,64,2,128,1,bfloat16,bfloat16,0,0.011435200273990632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,1,128,1,bfloat16,fp8,0,39.20010681152344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,1,128,1,fp8,fp8,0,38.32412414550781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,2,128,1,bfloat16,fp8,0,38.05122375488281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,2,128,1,fp8,fp8,0,38.57085266113281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,4,128,1,bfloat16,fp8,0,38.16871948242188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,1,128,1,bfloat16,bfloat16,0,45.85909729003906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,2,128,1,bfloat16,bfloat16,0,46.09715881347656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,4,128,1,bfloat16,bfloat16,0,46.735818481445314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,48,128,1,bfloat16,bfloat16,0,23.788909912109375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,1,128,1,bfloat16,bfloat16,0,23.848353576660156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,8,128,1,bfloat16,fp8,0,38.163937377929685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,4,128,1,fp8,fp8,0,39.41194458007813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,8,128,1,fp8,fp8,0,38.90449829101563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,48,8,128,1,bfloat16,bfloat16,0,46.220880126953126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,1,128,1,bfloat16,fp8,0,19.578172302246095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,1,128,1,fp8,fp8,0,20.091651916503906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,2,128,1,bfloat16,fp8,0,19.17647705078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,2,128,1,fp8,fp8,0,18.870401000976564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,2,128,1,bfloat16,bfloat16,0,22.301901245117186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,4,128,1,bfloat16,fp8,0,19.299154663085936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,4,128,1,bfloat16,bfloat16,0,22.262786865234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,4,128,1,fp8,fp8,0,20.824876403808595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,48,128,1,bfloat16,bfloat16,0,12.203323364257812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,8,128,1,bfloat16,fp8,0,20.194325256347657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,8,128,1,fp8,fp8,0,19.52401123046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,8,128,1,bfloat16,bfloat16,0,24.437985229492188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,1,128,1,bfloat16,fp8,0,9.5247314453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,1,128,1,bfloat16,bfloat16,0,11.764527893066406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,1,128,1,fp8,fp8,0,9.505891418457031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,2,128,1,bfloat16,bfloat16,0,11.24676971435547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,2,128,1,bfloat16,fp8,0,9.407943725585938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,2,128,1,fp8,fp8,0,9.457793426513671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,4,128,1,bfloat16,bfloat16,0,11.630542755126953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,4,128,1,bfloat16,fp8,0,9.624822235107422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,4,128,1,fp8,fp8,0,9.646673583984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,8,128,1,bfloat16,bfloat16,0,11.257466888427734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,8,128,1,bfloat16,fp8,0,10.153517150878907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,48,128,1,bfloat16,bfloat16,0,6.3280895233154295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,8,128,1,fp8,fp8,0,9.482991790771484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,1,128,1,bfloat16,bfloat16,0,5.906761550903321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,1,128,1,bfloat16,fp8,0,4.643524932861328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,1,128,1,fp8,fp8,0,4.755771255493164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,2,128,1,bfloat16,bfloat16,0,5.665380859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,2,128,1,bfloat16,fp8,0,4.653777694702148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,2,128,1,fp8,fp8,0,4.765851211547852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,4,128,1,bfloat16,bfloat16,0,5.854254531860351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,4,128,1,bfloat16,fp8,0,4.6697345733642575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,4,128,1,fp8,fp8,0,4.7332111358642575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,8,128,1,bfloat16,bfloat16,0,6.472198486328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,8,128,1,bfloat16,fp8,0,4.71593132019043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,8,128,1,fp8,fp8,0,5.256371307373047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,1,128,1,bfloat16,fp8,0,21.65473175048828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,1,128,1,bfloat16,bfloat16,0,26.1386962890625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,48,128,1,bfloat16,fp8,0,4.8077342987060545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,48,48,128,1,fp8,fp8,0,4.895321655273437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,48,128,1,fp8,fp8,0,9.634241485595703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,48,48,128,1,bfloat16,fp8,0,9.6169677734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,1,128,1,fp8,fp8,0,22.858378601074218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,2,128,1,bfloat16,bfloat16,0,26.525088500976562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,48,128,1,bfloat16,fp8,0,19.135009765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,48,48,128,1,fp8,fp8,0,19.16112518310547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,2,128,1,bfloat16,fp8,0,21.0363037109375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,2,128,1,fp8,fp8,0,21.5502685546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,4,128,1,bfloat16,fp8,0,21.220567321777345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,4,128,1,fp8,fp8,0,21.97889404296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,4,128,1,bfloat16,bfloat16,0,26.048617553710937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,8,128,1,bfloat16,fp8,0,23.066265869140626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,48,128,1,bfloat16,fp8,0,12.828697204589844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,48,128,1,bfloat16,bfloat16,0,15.452238464355469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,8,128,1,bfloat16,bfloat16,0,26.28941955566406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,48,8,128,1,fp8,fp8,0,23.237702941894533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,48,128,1,fp8,fp8,0,12.416334533691407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,1,128,1,bfloat16,bfloat16,0,13.751162719726562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,1,128,1,bfloat16,fp8,0,11.089424133300781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,1,128,1,fp8,fp8,0,10.586801910400391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,2,128,1,bfloat16,fp8,0,10.530912017822265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,2,128,1,bfloat16,bfloat16,0,13.114425659179688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,2,128,1,fp8,fp8,0,10.954698944091797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,4,128,1,bfloat16,fp8,0,10.728598022460938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,4,128,1,bfloat16,bfloat16,0,12.909494018554687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,4,128,1,fp8,fp8,0,10.723481750488281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,48,128,1,bfloat16,fp8,0,5.617681503295898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,48,128,1,bfloat16,bfloat16,0,6.8108879089355465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,8,128,1,bfloat16,bfloat16,0,12.94071044921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,48,128,1,fp8,fp8,0,5.522779083251953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,8,128,1,bfloat16,fp8,0,10.675398254394532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,1,128,1,bfloat16,bfloat16,0,6.171791839599609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,48,8,128,1,fp8,fp8,0,10.608796691894531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,1,128,1,bfloat16,fp8,0,5.3646080017089846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,1,128,1,fp8,fp8,0,5.405606460571289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,2,128,1,fp8,fp8,0,5.303852844238281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,2,128,1,bfloat16,fp8,0,5.755086517333984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,2,128,1,bfloat16,bfloat16,0,6.3626750946044925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,4,128,1,bfloat16,bfloat16,0,6.990617370605468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,4,128,1,bfloat16,fp8,0,5.399660873413086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,4,128,1,fp8,fp8,0,5.363363265991211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,8,128,1,bfloat16,bfloat16,0,6.8341522216796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,48,128,1,bfloat16,fp8,0,2.9589439392089845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,48,128,1,bfloat16,bfloat16,0,3.8671520233154295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,1,128,1,bfloat16,fp8,0,2.761065673828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,1,128,1,bfloat16,bfloat16,0,2.9536304473876953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,8,128,1,bfloat16,fp8,0,5.481304168701172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,48,128,1,fp8,fp8,0,3.152964782714844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,48,8,128,1,fp8,fp8,0,5.738337707519531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,1,128,1,fp8,fp8,0,2.8177791595458985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,2,128,1,bfloat16,fp8,0,2.9187456130981446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,2,128,1,bfloat16,bfloat16,0,3.3484798431396485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,4,128,1,bfloat16,fp8,0,2.718427276611328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,4,128,1,fp8,fp8,0,2.8513439178466795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,4,128,1,bfloat16,bfloat16,0,3.696664047241211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,8,128,1,bfloat16,bfloat16,0,3.4142513275146484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,8,128,1,bfloat16,fp8,0,2.7237775802612303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,8,128,1,fp8,fp8,0,2.852505683898926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,48,2,128,1,fp8,fp8,0,2.927817535400391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,1,128,1,fp8,fp8,0,15.0195068359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,1,128,1,bfloat16,fp8,0,15.855061340332032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,2,128,1,bfloat16,fp8,0,15.086740112304687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,2,128,1,fp8,fp8,0,15.838519287109374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,1,128,1,bfloat16,bfloat16,0,18.588079833984374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,4,128,1,bfloat16,fp8,0,14.862037658691406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,2,128,1,bfloat16,bfloat16,0,18.284652709960938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,4,128,1,bfloat16,bfloat16,0,18.506651306152342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,48,128,1,bfloat16,fp8,0,7.7645423889160154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,48,128,1,fp8,fp8,0,7.829444885253906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,48,128,1,bfloat16,bfloat16,0,9.708216094970703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,1,128,1,bfloat16,bfloat16,0,9.287544250488281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,4,128,1,fp8,fp8,0,15.275636291503906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,8,128,1,bfloat16,fp8,0,15.155210876464844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,8,128,1,fp8,fp8,0,15.558238220214843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,48,8,128,1,bfloat16,bfloat16,0,18.445916748046876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,1,128,1,bfloat16,fp8,0,7.379097747802734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,1,128,1,fp8,fp8,0,7.609811401367187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,2,128,1,bfloat16,bfloat16,0,9.282860565185548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,2,128,1,bfloat16,fp8,0,8.25353775024414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,2,128,1,fp8,fp8,0,8.089036560058593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,4,128,1,bfloat16,fp8,0,7.821683502197265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,4,128,1,bfloat16,bfloat16,0,10.447402954101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,4,128,1,fp8,fp8,0,7.7018882751464846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,48,128,1,bfloat16,bfloat16,0,4.983726501464844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,8,128,1,bfloat16,bfloat16,0,10.221873474121093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,48,128,1,bfloat16,fp8,0,4.145415878295898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,8,128,1,bfloat16,fp8,0,8.056630706787109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,48,128,1,fp8,fp8,0,3.9726768493652345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,48,8,128,1,fp8,fp8,0,8.162633514404297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,1,128,1,bfloat16,fp8,0,3.7357105255126952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,1,128,1,bfloat16,bfloat16,0,4.611977767944336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,1,128,1,fp8,fp8,0,3.8927230834960938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,2,128,1,bfloat16,bfloat16,0,4.526267242431641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,2,128,1,bfloat16,fp8,0,3.9240814208984376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,2,128,1,fp8,fp8,0,3.852652740478516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,4,128,1,bfloat16,fp8,0,3.992440032958984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,4,128,1,fp8,fp8,0,3.740732955932617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,4,128,1,bfloat16,bfloat16,0,4.464883041381836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,8,128,1,bfloat16,bfloat16,0,4.723168182373047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,48,128,1,bfloat16,fp8,0,2.028980827331543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,48,128,1,bfloat16,bfloat16,0,2.3698911666870117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,8,128,1,bfloat16,fp8,0,3.739683151245117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,48,128,1,fp8,fp8,0,2.135006332397461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,1,128,1,bfloat16,bfloat16,0,2.0986400604248048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,1,128,1,bfloat16,fp8,0,1.9709648132324218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,48,8,128,1,fp8,fp8,0,3.816664123535156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,1,128,1,fp8,fp8,0,2.0587072372436523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,2,128,1,bfloat16,bfloat16,0,2.173944091796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,2,128,1,fp8,fp8,0,1.8580640792846679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,2,128,1,bfloat16,fp8,0,2.237558364868164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,4,128,1,fp8,fp8,0,1.87193603515625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,4,128,1,bfloat16,fp8,0,2.0465839385986326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,8,128,1,bfloat16,bfloat16,0,2.0822320938110352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,4,128,1,bfloat16,bfloat16,0,2.684000015258789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,8,128,1,bfloat16,fp8,0,1.8860528945922852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,48,8,128,1,fp8,fp8,0,2.085276794433594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,1,128,1,bfloat16,fp8,0,19.860317993164063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,1,128,1,fp8,fp8,0,19.75725555419922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,2,128,1,bfloat16,fp8,0,19.337911987304686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,2,128,1,fp8,fp8,0,20.481204223632812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,4,128,1,bfloat16,fp8,0,20.694786071777344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,1,128,1,bfloat16,bfloat16,0,23.704081726074218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,2,128,1,bfloat16,bfloat16,0,23.94438781738281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,4,128,1,bfloat16,bfloat16,0,23.945751953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,48,128,1,bfloat16,fp8,0,10.614254760742188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,48,128,1,fp8,fp8,0,10.530363464355469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,48,128,1,bfloat16,bfloat16,0,13.54599609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,1,128,1,bfloat16,bfloat16,0,12.082892608642577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,4,128,1,fp8,fp8,0,20.7714599609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,8,128,1,bfloat16,fp8,0,20.524742126464844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,8,128,1,fp8,fp8,0,20.90529022216797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,1,128,1,bfloat16,fp8,0,9.6828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,48,8,128,1,bfloat16,bfloat16,0,23.983460998535158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,1,128,1,fp8,fp8,0,9.616935729980469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,2,128,1,bfloat16,fp8,0,9.725154876708984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,2,128,1,bfloat16,bfloat16,0,11.69561767578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,2,128,1,fp8,fp8,0,9.864582061767578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,4,128,1,bfloat16,bfloat16,0,11.531015777587891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,4,128,1,bfloat16,fp8,0,10.410419464111328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,4,128,1,fp8,fp8,0,9.98315200805664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,48,128,1,bfloat16,bfloat16,0,6.514173126220703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,8,128,1,bfloat16,fp8,0,9.818321228027344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,8,128,1,bfloat16,bfloat16,0,11.728839874267578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,48,128,1,bfloat16,fp8,0,5.337409591674804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,48,8,128,1,fp8,fp8,0,10.21539077758789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,48,128,1,fp8,fp8,0,5.156507110595703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,1,128,1,bfloat16,bfloat16,0,5.920939254760742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,1,128,1,bfloat16,fp8,0,5.093272018432617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,1,128,1,fp8,fp8,0,5.151504135131836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,2,128,1,fp8,fp8,0,5.052438354492187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,2,128,1,bfloat16,fp8,0,5.153964614868164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,2,128,1,bfloat16,bfloat16,0,6.978249359130859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,4,128,1,bfloat16,fp8,0,5.020721435546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,4,128,1,bfloat16,bfloat16,0,6.138711929321289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,4,128,1,fp8,fp8,0,5.290393447875976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,48,128,1,bfloat16,bfloat16,0,3.171566390991211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,48,128,1,bfloat16,fp8,0,2.861356735229492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,8,128,1,bfloat16,fp8,0,5.218207931518554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,8,128,1,bfloat16,bfloat16,0,6.152908706665039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,48,128,1,fp8,fp8,0,2.642857551574707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,48,8,128,1,fp8,fp8,0,5.121185684204102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,1,128,1,bfloat16,bfloat16,0,3.025927925109863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,2,128,1,bfloat16,fp8,0,2.5123104095458983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,1,128,1,fp8,fp8,0,2.8364912033081056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,2,128,1,fp8,fp8,0,2.623574447631836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,4,128,1,bfloat16,bfloat16,0,2.8946271896362306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,4,128,1,bfloat16,fp8,0,2.5478511810302735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,4,128,1,fp8,fp8,0,2.644113540649414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,8,128,1,bfloat16,fp8,0,2.4451871871948243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,8,128,1,bfloat16,bfloat16,0,2.9664432525634767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,48,128,1,bfloat16,fp8,0,1.3419376373291017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,8,128,1,fp8,fp8,0,2.445591926574707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,48,128,1,bfloat16,bfloat16,0,1.6144912719726563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,48,128,1,fp8,fp8,0,1.2974703788757325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,1,128,1,bfloat16,fp8,0,2.912678337097168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,48,2,128,1,bfloat16,bfloat16,0,2.8658624649047852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,1,128,1,bfloat16,bfloat16,0,1.33919038772583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,1,128,1,fp8,fp8,0,1.2603535652160645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,1,128,1,bfloat16,fp8,0,1.4155072212219237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,2,128,1,bfloat16,bfloat16,0,1.3841343879699708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,2,128,1,bfloat16,fp8,0,1.3575375556945801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,2,128,1,fp8,fp8,0,1.2152912139892578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,4,128,1,bfloat16,bfloat16,0,1.3253775596618653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,4,128,1,bfloat16,fp8,0,1.2435728073120118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,4,128,1,fp8,fp8,0,1.2736031532287597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,8,128,1,bfloat16,bfloat16,0,1.3775856018066406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,8,128,1,fp8,fp8,0,1.2646800041198731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,48,8,128,1,bfloat16,fp8,0,1.3973487854003905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,1,128,1,fp8,fp8,0,11.155305480957031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,2,128,1,fp8,fp8,0,11.178553771972656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,1,128,1,bfloat16,fp8,0,11.942494201660157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,2,128,1,bfloat16,fp8,0,11.458268737792968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,1,128,1,bfloat16,bfloat16,0,13.390187072753907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,2,128,1,bfloat16,bfloat16,0,13.703610229492188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,4,128,1,bfloat16,fp8,0,11.251110076904297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,4,128,1,bfloat16,bfloat16,0,13.246343994140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,48,128,1,bfloat16,fp8,0,6.190644836425781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,4,128,1,fp8,fp8,0,11.243582153320313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,48,128,1,fp8,fp8,0,6.020312118530273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,1,128,1,bfloat16,bfloat16,0,6.442616271972656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,8,128,1,bfloat16,fp8,0,12.336824035644531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,48,128,1,bfloat16,bfloat16,0,7.717763519287109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,8,128,1,fp8,fp8,0,12.070582580566406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,48,8,128,1,bfloat16,bfloat16,0,14.043260192871093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,1,128,1,bfloat16,fp8,0,5.761508941650391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,1,128,1,fp8,fp8,0,5.570576095581055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,2,128,1,bfloat16,bfloat16,0,6.567700958251953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,2,128,1,bfloat16,fp8,0,5.69276008605957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,2,128,1,fp8,fp8,0,6.038057708740235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,4,128,1,bfloat16,fp8,0,5.9198352813720705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,4,128,1,bfloat16,bfloat16,0,6.692801666259766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,4,128,1,fp8,fp8,0,5.617814254760742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,48,128,1,bfloat16,fp8,0,3.200579071044922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,48,128,1,fp8,fp8,0,3.135148811340332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,48,128,1,bfloat16,bfloat16,0,3.716572952270508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,8,128,1,bfloat16,bfloat16,0,6.848977661132812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,1,128,1,bfloat16,bfloat16,0,3.260726547241211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,8,128,1,bfloat16,fp8,0,6.462518310546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,1,128,1,bfloat16,fp8,0,2.816022491455078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,48,8,128,1,fp8,fp8,0,5.831155014038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,1,128,1,fp8,fp8,0,2.985691261291504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,2,128,1,bfloat16,fp8,0,3.101750373840332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,2,128,1,bfloat16,bfloat16,0,3.4913761138916017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,2,128,1,fp8,fp8,0,3.2423168182373048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,4,128,1,bfloat16,bfloat16,0,3.197348785400391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,4,128,1,bfloat16,fp8,0,3.148910331726074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,4,128,1,fp8,fp8,0,2.8196672439575194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,48,128,1,bfloat16,fp8,0,1.5309311866760253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,48,128,1,bfloat16,bfloat16,0,1.9709535598754884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,48,128,1,fp8,fp8,0,1.642568016052246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,1,128,1,bfloat16,bfloat16,0,1.6281167984008789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,8,128,1,bfloat16,bfloat16,0,3.8958366394042967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,1,128,1,bfloat16,fp8,0,1.4014304161071778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,8,128,1,bfloat16,fp8,0,3.034644889831543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,48,8,128,1,fp8,fp8,0,3.152644729614258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,1,128,1,fp8,fp8,0,1.4014224052429198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,2,128,1,bfloat16,fp8,0,1.4257967948913575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,2,128,1,fp8,fp8,0,1.5759167671203613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,2,128,1,bfloat16,bfloat16,0,1.7397039413452149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,4,128,1,bfloat16,fp8,0,1.4139616012573242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,4,128,1,fp8,fp8,0,1.4920656204223632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,4,128,1,bfloat16,bfloat16,0,2.037998390197754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,8,128,1,bfloat16,fp8,0,1.424168014526367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,8,128,1,bfloat16,bfloat16,0,1.63067684173584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,48,128,1,bfloat16,bfloat16,0,0.8968031883239747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,48,128,1,bfloat16,fp8,0,0.9054384231567383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,48,128,1,fp8,fp8,0,0.7670127868652343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,48,8,128,1,fp8,fp8,0,1.4235072135925293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,1,128,1,bfloat16,bfloat16,0,0.8763104438781738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,1,128,1,bfloat16,fp8,0,0.9773311614990234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,1,128,1,fp8,fp8,0,0.7049839973449707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,2,128,1,bfloat16,fp8,0,0.7497375965118408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,4,128,1,bfloat16,bfloat16,0,0.7568880081176758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,2,128,1,fp8,fp8,0,0.8268272399902343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,2,128,1,bfloat16,bfloat16,0,0.8648528099060059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,4,128,1,bfloat16,fp8,0,0.7120240211486817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,4,128,1,fp8,fp8,0,0.8233072280883789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,8,128,1,bfloat16,bfloat16,0,0.8580176353454589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,8,128,1,bfloat16,fp8,0,0.7161695957183838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,48,8,128,1,fp8,fp8,0,0.7245007991790772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,1,128,1,bfloat16,fp8,0,10.541632080078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,1,128,1,fp8,fp8,0,10.476364898681641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,2,128,1,fp8,fp8,0,10.51031036376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,4,128,1,bfloat16,fp8,0,10.682418823242188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,2,128,1,bfloat16,fp8,0,11.284307098388672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,1,128,1,bfloat16,bfloat16,0,12.362091064453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,2,128,1,bfloat16,bfloat16,0,12.63753890991211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,4,128,1,bfloat16,bfloat16,0,12.900604248046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,48,128,1,bfloat16,fp8,0,6.071662521362304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,48,128,1,bfloat16,bfloat16,0,7.39720458984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,48,128,1,fp8,fp8,0,6.50353775024414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,1,128,1,bfloat16,bfloat16,0,6.256497573852539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,4,128,1,fp8,fp8,0,11.18786849975586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,8,128,1,bfloat16,fp8,0,10.665058898925782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,8,128,1,fp8,fp8,0,10.658640289306641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,48,8,128,1,bfloat16,bfloat16,0,13.090345764160157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,1,128,1,bfloat16,fp8,0,5.381859207153321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,1,128,1,fp8,fp8,0,5.581571197509765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,2,128,1,bfloat16,fp8,0,5.372635269165039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,2,128,1,bfloat16,bfloat16,0,6.33374252319336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,2,128,1,fp8,fp8,0,5.380992126464844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,4,128,1,bfloat16,fp8,0,5.683179092407227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,4,128,1,bfloat16,bfloat16,0,6.367841720581055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,4,128,1,fp8,fp8,0,5.481280136108398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,48,128,1,bfloat16,bfloat16,0,3.7242961883544923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,8,128,1,bfloat16,fp8,0,5.668686294555664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,48,128,1,bfloat16,fp8,0,3.0611839294433594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,8,128,1,fp8,fp8,0,5.376278305053711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,48,8,128,1,bfloat16,bfloat16,0,6.571311950683594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,48,128,1,fp8,fp8,0,3.274078369140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,1,128,1,bfloat16,bfloat16,0,3.091739273071289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,1,128,1,fp8,fp8,0,2.6420751571655274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,2,128,1,bfloat16,fp8,0,2.635406494140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,1,128,1,bfloat16,fp8,0,2.9610927581787108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,2,128,1,bfloat16,bfloat16,0,3.204391860961914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,2,128,1,fp8,fp8,0,2.9162128448486326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,4,128,1,bfloat16,fp8,0,2.8344688415527344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,4,128,1,bfloat16,bfloat16,0,2.95947208404541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,4,128,1,fp8,fp8,0,2.9781824111938477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,48,128,1,bfloat16,fp8,0,1.5083776473999024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,48,128,1,bfloat16,bfloat16,0,1.8827280044555663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,48,128,1,fp8,fp8,0,1.5461872100830079
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,1,128,1,bfloat16,bfloat16,0,1.6045087814331054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,8,128,1,bfloat16,fp8,0,2.6729679107666016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,1,128,1,bfloat16,fp8,0,1.3612560272216796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,8,128,1,bfloat16,bfloat16,0,3.2592960357666017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,48,8,128,1,fp8,fp8,0,2.7956192016601564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,1,128,1,fp8,fp8,0,1.3536208152770997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,2,128,1,bfloat16,fp8,0,1.3452896118164062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,2,128,1,fp8,fp8,0,1.3828415870666504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,2,128,1,bfloat16,bfloat16,0,1.5743951797485352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,4,128,1,bfloat16,fp8,0,1.334455966949463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,4,128,1,bfloat16,bfloat16,0,1.431668758392334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,8,128,1,bfloat16,bfloat16,0,1.4618495941162108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,4,128,1,fp8,fp8,0,1.5470895767211914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,48,128,1,bfloat16,fp8,0,0.8169296264648438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,8,128,1,bfloat16,fp8,0,1.384832000732422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,1,128,1,bfloat16,bfloat16,0,0.7610511779785156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,48,128,1,fp8,fp8,0,0.7457104206085206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,48,8,128,1,fp8,fp8,0,1.3488384246826173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,1,128,1,bfloat16,fp8,0,0.6766655921936036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,1,128,1,fp8,fp8,0,0.7167967796325684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,2,128,1,bfloat16,fp8,0,0.6873983860015869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,2,128,1,bfloat16,bfloat16,0,0.7282400131225586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,4,128,1,bfloat16,bfloat16,0,0.7196144104003906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,4,128,1,bfloat16,fp8,0,0.6778960227966309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,4,128,1,fp8,fp8,0,0.7230591773986816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,8,128,1,bfloat16,bfloat16,0,0.745249605178833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,8,128,1,bfloat16,fp8,0,0.6853504180908203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,48,128,1,bfloat16,fp8,0,0.38270881175994875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,48,128,1,bfloat16,bfloat16,0,0.4311567783355713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,48,128,1,fp8,fp8,0,0.4111743927001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,1,128,1,bfloat16,bfloat16,0,0.364302396774292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,1,128,1,bfloat16,fp8,0,0.33879361152648924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,1,128,1,fp8,fp8,0,0.3411040067672729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,2,128,1,bfloat16,bfloat16,0,0.36405119895935056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,2,128,1,bfloat16,fp8,0,0.36320641040802004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,2,128,1,fp8,fp8,0,0.3471983909606934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,4,128,1,bfloat16,bfloat16,0,0.3674400091171265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,4,128,1,bfloat16,fp8,0,0.3408416032791138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,4,128,1,fp8,fp8,0,0.37208640575408936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,8,128,1,bfloat16,fp8,0,0.3479487895965576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,8,128,1,bfloat16,bfloat16,0,0.38008320331573486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,48,8,128,1,fp8,fp8,0,0.3918159961700439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,48,128,1,bfloat16,bfloat16,0,0.846288013458252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,2,128,1,fp8,fp8,0,0.8174752235412598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,48,8,128,1,fp8,fp8,0,0.7436336040496826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,1,128,1,bfloat16,fp8,0,6.273467254638672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,1,128,1,fp8,fp8,0,6.3153423309326175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,2,128,1,bfloat16,fp8,0,6.542192077636718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,1,128,1,bfloat16,bfloat16,0,7.449505615234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,2,128,1,fp8,fp8,0,6.229486465454102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,4,128,1,bfloat16,fp8,0,6.368998336791992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,4,128,1,bfloat16,bfloat16,0,7.375704193115235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,2,128,1,bfloat16,bfloat16,0,7.385337829589844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,48,128,1,bfloat16,bfloat16,0,4.311280059814453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,4,128,1,fp8,fp8,0,6.2833984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,48,128,1,bfloat16,fp8,0,3.5878017425537108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,48,128,1,fp8,fp8,0,3.619852828979492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,8,128,1,fp8,fp8,0,6.365518569946289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,8,128,1,bfloat16,fp8,0,6.908128356933593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,48,8,128,1,bfloat16,bfloat16,0,7.6617790222167965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,1,128,1,bfloat16,bfloat16,0,3.639446258544922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,1,128,1,bfloat16,fp8,0,3.1766576766967773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,1,128,1,fp8,fp8,0,3.1074399948120117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,2,128,1,bfloat16,fp8,0,3.1262399673461916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,2,128,1,bfloat16,bfloat16,0,3.6313838958740234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,2,128,1,fp8,fp8,0,3.1198192596435548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,4,128,1,bfloat16,fp8,0,3.307798385620117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,4,128,1,bfloat16,bfloat16,0,3.8493583679199217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,4,128,1,fp8,fp8,0,3.255313491821289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,8,128,1,bfloat16,bfloat16,0,3.514303970336914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,48,128,1,bfloat16,fp8,0,1.7939472198486328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,48,128,1,bfloat16,bfloat16,0,2.1772607803344726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,8,128,1,bfloat16,fp8,0,3.1800256729125977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,48,128,1,fp8,fp8,0,1.7988079071044922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,1,128,1,bfloat16,bfloat16,0,1.77728328704834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,48,8,128,1,fp8,fp8,0,3.1883167266845702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,2,128,1,bfloat16,bfloat16,0,1.6856992721557618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,1,128,1,fp8,fp8,0,1.7219039916992187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,2,128,1,bfloat16,fp8,0,1.6911376953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,2,128,1,fp8,fp8,0,1.568393611907959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,4,128,1,bfloat16,bfloat16,0,1.7819599151611327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,4,128,1,bfloat16,fp8,0,1.5801103591918946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,4,128,1,fp8,fp8,0,1.7046432495117188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,8,128,1,bfloat16,bfloat16,0,1.7317279815673827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,48,128,1,bfloat16,bfloat16,0,1.1343168258666991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,8,128,1,bfloat16,fp8,0,1.60665283203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,48,128,1,bfloat16,fp8,0,0.9073856353759766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,8,128,1,fp8,fp8,0,1.5963711738586426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,1,128,1,fp8,fp8,0,0.7887919902801513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,1,128,1,bfloat16,bfloat16,0,0.8915920257568359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,48,1,128,1,bfloat16,fp8,0,1.5630576133728027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,2,128,1,bfloat16,bfloat16,0,0.8350576400756836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,1,128,1,bfloat16,fp8,0,0.9267359733581543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,2,128,1,fp8,fp8,0,0.7911087989807128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,4,128,1,bfloat16,fp8,0,0.7944303989410401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,4,128,1,bfloat16,bfloat16,0,0.8903599739074707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,4,128,1,fp8,fp8,0,0.8957200050354004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,8,128,1,bfloat16,bfloat16,0,0.9375375747680664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,48,128,1,bfloat16,bfloat16,0,0.5517216205596924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,48,128,1,bfloat16,fp8,0,0.48723678588867186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,48,128,1,fp8,fp8,0,0.521884822845459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,8,128,1,bfloat16,fp8,0,0.8085488319396973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,8,128,1,fp8,fp8,0,0.8075183868408203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,1,128,1,bfloat16,bfloat16,0,0.4415071964263916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,1,128,1,bfloat16,fp8,0,0.41829118728637693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,1,128,1,fp8,fp8,0,0.4451136112213135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,2,128,1,bfloat16,bfloat16,0,0.42977280616760255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,2,128,1,bfloat16,fp8,0,0.4008639812469482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,2,128,1,fp8,fp8,0,0.41811041831970214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,4,128,1,bfloat16,bfloat16,0,0.44128642082214353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,4,128,1,bfloat16,fp8,0,0.4349376201629639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,4,128,1,fp8,fp8,0,0.4114223957061768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,8,128,1,bfloat16,bfloat16,0,0.4387712001800537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,8,128,1,bfloat16,fp8,0,0.4209280014038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,48,128,1,bfloat16,fp8,0,0.23930881023406983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,48,128,1,bfloat16,bfloat16,0,0.2868272066116333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,48,8,128,1,fp8,fp8,0,0.41614718437194825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,1,128,1,bfloat16,bfloat16,0,0.2212671995162964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,1,128,1,fp8,fp8,0,0.22133920192718506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,2,128,1,bfloat16,bfloat16,0,0.22096478939056396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,48,128,1,fp8,fp8,0,0.9574543952941894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,48,2,128,1,bfloat16,fp8,0,0.7877408027648926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,2,128,1,bfloat16,fp8,0,0.20660159587860108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,2,128,1,fp8,fp8,0,0.20998079776763917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,4,128,1,bfloat16,fp8,0,0.21020801067352296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,4,128,1,fp8,fp8,0,0.20803840160369874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,8,128,1,bfloat16,bfloat16,0,0.22179360389709474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,8,128,1,bfloat16,fp8,0,0.2103503942489624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,8,128,1,fp8,fp8,0,0.2137376070022583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,48,128,1,fp8,fp8,0,0.2349071979522705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,1,128,1,bfloat16,fp8,0,0.20977919101715087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,48,4,128,1,bfloat16,bfloat16,0,0.22833120822906494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,1,128,1,bfloat16,fp8,0,6.116795349121094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,1,128,1,fp8,fp8,0,6.115340805053711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,2,128,1,bfloat16,fp8,0,6.213875198364258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,1,128,1,bfloat16,bfloat16,0,6.982514953613281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,2,128,1,bfloat16,bfloat16,0,7.013329315185547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,2,128,1,fp8,fp8,0,6.181475067138672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,4,128,1,bfloat16,fp8,0,6.226556777954102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,4,128,1,bfloat16,bfloat16,0,7.152201843261719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,48,128,1,bfloat16,bfloat16,0,4.487481689453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,4,128,1,fp8,fp8,0,6.179369735717773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,8,128,1,bfloat16,fp8,0,6.2940208435058596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,48,128,1,bfloat16,fp8,0,3.664044952392578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,8,128,1,bfloat16,bfloat16,0,7.411524963378906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,48,8,128,1,fp8,fp8,0,6.5007789611816404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,1,128,1,bfloat16,bfloat16,0,3.3820751190185545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,48,128,1,fp8,fp8,0,3.655180740356445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,1,128,1,bfloat16,fp8,0,3.1843088150024412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,1,128,1,fp8,fp8,0,3.0411487579345704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,2,128,1,bfloat16,bfloat16,0,3.2716880798339845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,2,128,1,bfloat16,fp8,0,3.0741167068481445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,2,128,1,fp8,fp8,0,3.108403205871582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,4,128,1,bfloat16,fp8,0,3.1167600631713865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,4,128,1,fp8,fp8,0,3.0945199966430663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,4,128,1,bfloat16,bfloat16,0,3.5665615081787108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,48,128,1,bfloat16,bfloat16,0,2.178745651245117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,1,128,1,bfloat16,bfloat16,0,1.6705488204956054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,8,128,1,bfloat16,fp8,0,3.1404655456542967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,48,128,1,bfloat16,fp8,0,1.8332096099853517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,8,128,1,bfloat16,bfloat16,0,3.6440929412841796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,48,128,1,fp8,fp8,0,1.8345375061035156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,1,128,1,bfloat16,fp8,0,1.6430736541748048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,48,8,128,1,fp8,fp8,0,3.2359855651855467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,1,128,1,fp8,fp8,0,1.694910430908203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,2,128,1,bfloat16,fp8,0,1.5294431686401366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,2,128,1,fp8,fp8,0,1.5392255783081055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,4,128,1,bfloat16,fp8,0,1.552030372619629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,2,128,1,bfloat16,bfloat16,0,1.7564016342163087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,4,128,1,fp8,fp8,0,1.5598352432250977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,4,128,1,bfloat16,bfloat16,0,1.7852848052978516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,8,128,1,bfloat16,bfloat16,0,1.74847354888916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,1,128,1,bfloat16,bfloat16,0,0.8097104072570801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,48,128,1,bfloat16,fp8,0,0.9272656440734863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,1,128,1,bfloat16,fp8,0,0.8572159767150879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,48,128,1,bfloat16,bfloat16,0,1.0778287887573241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,48,128,1,fp8,fp8,0,0.9847184181213379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,8,128,1,fp8,fp8,0,1.571561622619629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,48,8,128,1,bfloat16,fp8,0,1.6339487075805663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,1,128,1,fp8,fp8,0,0.8520383834838867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,2,128,1,bfloat16,fp8,0,0.7749792098999023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,2,128,1,bfloat16,bfloat16,0,0.8955007553100586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,2,128,1,fp8,fp8,0,0.7997759819030762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,4,128,1,bfloat16,bfloat16,0,0.8300512313842774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,4,128,1,bfloat16,fp8,0,0.779857587814331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,4,128,1,fp8,fp8,0,0.7854879856109619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,8,128,1,bfloat16,bfloat16,0,0.8536975860595704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,48,128,1,bfloat16,fp8,0,0.47668957710266113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,8,128,1,bfloat16,fp8,0,0.8034159660339355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,48,8,128,1,fp8,fp8,0,0.8505104064941407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,1,128,1,bfloat16,bfloat16,0,0.41476640701293943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,48,128,1,fp8,fp8,0,0.47557759284973145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,1,128,1,bfloat16,fp8,0,0.39435999393463134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,1,128,1,fp8,fp8,0,0.3935199975967407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,2,128,1,bfloat16,bfloat16,0,0.42127838134765627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,2,128,1,bfloat16,fp8,0,0.4193840026855469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,2,128,1,fp8,fp8,0,0.39690399169921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,4,128,1,bfloat16,bfloat16,0,0.42898077964782716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,4,128,1,bfloat16,fp8,0,0.4000256061553955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,4,128,1,fp8,fp8,0,0.3990799903869629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,8,128,1,bfloat16,bfloat16,0,0.4366191864013672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,8,128,1,bfloat16,fp8,0,0.418012809753418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,48,128,1,bfloat16,fp8,0,0.24361441135406495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,48,128,1,bfloat16,bfloat16,0,0.28504319190979005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,8,128,1,fp8,fp8,0,0.40798401832580566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,1,128,1,bfloat16,bfloat16,0,0.2164223909378052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,1,128,1,bfloat16,fp8,0,0.20625920295715333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,1,128,1,fp8,fp8,0,0.20138559341430665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,2,128,1,bfloat16,bfloat16,0,0.21860320568084718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,2,128,1,bfloat16,fp8,0,0.20289440155029298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,2,128,1,fp8,fp8,0,0.20816481113433838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,4,128,1,bfloat16,bfloat16,0,0.2183759927749634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,4,128,1,bfloat16,fp8,0,0.20425760746002197
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,4,128,1,fp8,fp8,0,0.21096320152282716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,8,128,1,bfloat16,bfloat16,0,0.22347838878631593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,8,128,1,bfloat16,fp8,0,0.2123647928237915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,48,128,1,bfloat16,bfloat16,0,0.14582879543304444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,48,128,1,bfloat16,fp8,0,0.131113600730896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,48,128,1,fp8,fp8,0,0.12689440250396727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,1,128,1,bfloat16,bfloat16,0,0.11394399404525757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,1,128,1,bfloat16,fp8,0,0.10522240400314331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,1,128,1,fp8,fp8,0,0.10858880281448365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,2,128,1,bfloat16,bfloat16,0,0.1132464051246643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,2,128,1,bfloat16,fp8,0,0.10870879888534546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,2,128,1,fp8,fp8,0,0.10517439842224122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,4,128,1,bfloat16,bfloat16,0,0.11684479713439941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,4,128,1,bfloat16,fp8,0,0.1067728042602539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,4,128,1,fp8,fp8,0,0.11002559661865234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,8,128,1,bfloat16,bfloat16,0,0.11477119922637939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,8,128,1,bfloat16,fp8,0,0.11301280260086059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,48,8,128,1,fp8,fp8,0,0.11191200017929077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,48,48,128,1,bfloat16,bfloat16,0,0.5461056232452393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,48,128,1,fp8,fp8,0,0.24358561038970947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,48,8,128,1,fp8,fp8,0,0.21031520366668702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,1,128,1,bfloat16,fp8,0,3.7537567138671877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,1,128,1,fp8,fp8,0,3.863686370849609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,1,128,1,bfloat16,bfloat16,0,4.2006591796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,2,128,1,bfloat16,bfloat16,0,4.222211074829102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,2,128,1,bfloat16,fp8,0,3.8115745544433595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,2,128,1,fp8,fp8,0,3.8383312225341797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,4,128,1,bfloat16,bfloat16,0,4.260871887207031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,4,128,1,bfloat16,fp8,0,3.9115665435791014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,48,128,1,bfloat16,bfloat16,0,2.7807647705078127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,48,128,1,bfloat16,fp8,0,2.3721471786499024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,4,128,1,fp8,fp8,0,3.9086673736572264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,8,128,1,bfloat16,fp8,0,3.9248897552490236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,8,128,1,fp8,fp8,0,3.944571304321289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,48,8,128,1,bfloat16,bfloat16,0,4.491110229492188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,1,128,1,bfloat16,bfloat16,0,1.9604000091552733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,48,128,1,fp8,fp8,0,2.3796735763549806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,1,128,1,fp8,fp8,0,1.8913423538208007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,1,128,1,bfloat16,fp8,0,2.0225887298583984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,2,128,1,bfloat16,bfloat16,0,2.004803276062012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,2,128,1,bfloat16,fp8,0,1.9229391098022461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,2,128,1,fp8,fp8,0,1.9406431198120118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,4,128,1,bfloat16,bfloat16,0,2.2262079238891603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,4,128,1,bfloat16,fp8,0,2.0648591995239256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,4,128,1,fp8,fp8,0,1.9192319869995118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,48,128,1,bfloat16,bfloat16,0,1.3874383926391602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,8,128,1,bfloat16,bfloat16,0,2.0778560638427734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,48,128,1,bfloat16,fp8,0,1.1961088180541992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,8,128,1,fp8,fp8,0,1.995030403137207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,48,128,1,fp8,fp8,0,1.229576015472412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,1,128,1,bfloat16,fp8,0,0.9555999755859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,1,128,1,bfloat16,bfloat16,0,1.0210207939147948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,1,128,1,fp8,fp8,0,1.012603187561035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,2,128,1,bfloat16,bfloat16,0,1.0916111946105957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,2,128,1,bfloat16,fp8,0,0.9600704193115235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,2,128,1,fp8,fp8,0,0.9602095603942871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,4,128,1,bfloat16,bfloat16,0,1.0148991584777831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,4,128,1,bfloat16,fp8,0,0.9655584335327149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,4,128,1,fp8,fp8,0,0.983347225189209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,8,128,1,bfloat16,bfloat16,0,1.0525391578674317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,48,128,1,bfloat16,bfloat16,0,0.6915152072906494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,1,128,1,bfloat16,bfloat16,0,0.49886398315429686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,8,128,1,fp8,fp8,0,0.9887791633605957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,48,128,1,fp8,fp8,0,0.6025199890136719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,1,128,1,bfloat16,fp8,0,0.5208687782287598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,1,128,1,fp8,fp8,0,0.4840703964233398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,2,128,1,bfloat16,fp8,0,0.4840720176696777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,2,128,1,fp8,fp8,0,0.4934512138366699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,4,128,1,bfloat16,bfloat16,0,0.5274032115936279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,4,128,1,bfloat16,fp8,0,0.4893343925476074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,48,8,128,1,bfloat16,fp8,0,2.003094482421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,4,128,1,fp8,fp8,0,0.49309120178222654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,8,128,1,bfloat16,bfloat16,0,0.539463996887207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,8,128,1,bfloat16,fp8,0,0.49988322257995604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,8,128,1,fp8,fp8,0,0.5014143943786621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,48,128,1,bfloat16,bfloat16,0,0.3561088085174561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,48,128,1,bfloat16,fp8,0,0.3152559995651245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,48,128,1,fp8,fp8,0,0.3090751886367798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,1,128,1,bfloat16,bfloat16,0,0.2576479911804199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,1,128,1,bfloat16,fp8,0,0.24738719463348388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,1,128,1,fp8,fp8,0,0.2534672021865845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,2,128,1,bfloat16,bfloat16,0,0.2579535961151123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,2,128,1,bfloat16,fp8,0,0.25025439262390137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,2,128,1,fp8,fp8,0,0.24871358871459961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,4,128,1,bfloat16,bfloat16,0,0.2682352066040039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,4,128,1,bfloat16,fp8,0,0.2511359930038452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,4,128,1,fp8,fp8,0,0.252455997467041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,8,128,1,bfloat16,bfloat16,0,0.2699984073638916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,8,128,1,bfloat16,fp8,0,0.2626703977584839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,48,8,128,1,fp8,fp8,0,0.2616944074630737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,48,128,1,bfloat16,fp8,0,0.1620336055755615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,48,128,1,fp8,fp8,0,0.16650400161743165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,1,128,1,bfloat16,bfloat16,0,0.1376960039138794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,1,128,1,bfloat16,fp8,0,0.12973599433898925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,48,8,128,1,bfloat16,fp8,0,1.0876015663146972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,48,128,1,bfloat16,fp8,0,0.6246335983276368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,1,128,1,fp8,fp8,0,0.1339295983314514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,2,128,1,bfloat16,bfloat16,0,0.13854240179061889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,2,128,1,bfloat16,fp8,0,0.13041599988937377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,2,128,1,fp8,fp8,0,0.13086559772491455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,4,128,1,bfloat16,bfloat16,0,0.14056320190429689
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,4,128,1,bfloat16,fp8,0,0.1351359963417053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,4,128,1,fp8,fp8,0,0.13465919494628906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,8,128,1,bfloat16,bfloat16,0,0.1410688042640686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,8,128,1,bfloat16,fp8,0,0.13460479974746703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,48,128,1,bfloat16,bfloat16,0,0.10061440467834473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,48,2,128,1,bfloat16,bfloat16,0,0.5212528228759765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,1,128,1,bfloat16,bfloat16,0,0.0719648003578186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,1,128,1,bfloat16,fp8,0,0.07041760087013245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,1,128,1,fp8,fp8,0,0.06869120001792908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,2,128,1,bfloat16,bfloat16,0,0.07319679856300354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,2,128,1,bfloat16,fp8,0,0.0702992022037506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,2,128,1,fp8,fp8,0,0.06926559805870056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,4,128,1,bfloat16,bfloat16,0,0.07539359927177429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,4,128,1,bfloat16,fp8,0,0.07116479873657226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,4,128,1,fp8,fp8,0,0.06986399888992309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,8,128,1,bfloat16,bfloat16,0,0.07738879919052125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,8,128,1,bfloat16,fp8,0,0.07248160243034363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,8,128,1,fp8,fp8,0,0.07167999744415283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,48,128,1,bfloat16,bfloat16,0,0.1878543972969055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,48,8,128,1,fp8,fp8,0,0.138155198097229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,48,128,1,fp8,fp8,0,0.08751519918441772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,1,128,1,bfloat16,bfloat16,0,4.080971145629883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,1,128,1,bfloat16,fp8,0,4.053384017944336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,2,128,1,bfloat16,bfloat16,0,4.21601448059082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,48,48,128,1,bfloat16,fp8,0,0.08975039720535279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,2,128,1,bfloat16,fp8,0,4.040079879760742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,2,128,1,fp8,fp8,0,4.066089630126953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,1,128,1,fp8,fp8,0,4.006414413452148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,4,128,1,bfloat16,fp8,0,4.022073745727539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,4,128,1,fp8,fp8,0,4.066759872436523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,4,128,1,bfloat16,bfloat16,0,4.358204650878906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,48,128,1,bfloat16,bfloat16,0,3.105401611328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,48,128,1,bfloat16,fp8,0,2.6190351486206054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,8,128,1,bfloat16,bfloat16,0,4.505456161499024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,1,128,1,bfloat16,fp8,0,2.0148479461669924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,1,128,1,bfloat16,bfloat16,0,2.0481456756591796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,8,128,1,bfloat16,fp8,0,4.17555046081543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,48,128,1,fp8,fp8,0,2.600761604309082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,48,8,128,1,fp8,fp8,0,4.132575988769531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,1,128,1,fp8,fp8,0,2.0697504043579102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,2,128,1,bfloat16,bfloat16,0,2.100651168823242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,2,128,1,fp8,fp8,0,1.9925056457519532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,4,128,1,bfloat16,fp8,0,2.019191932678223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,2,128,1,bfloat16,fp8,0,2.1320367813110352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,4,128,1,fp8,fp8,0,2.0082799911499025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,8,128,1,bfloat16,bfloat16,0,2.182931137084961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,1,128,1,bfloat16,bfloat16,0,1.0183008193969727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,48,128,1,bfloat16,fp8,0,1.3071807861328124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,48,128,1,bfloat16,bfloat16,0,1.5581055641174317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,48,128,1,fp8,fp8,0,1.3503472328186035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,8,128,1,bfloat16,fp8,0,2.1037952423095705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,8,128,1,fp8,fp8,0,2.0799535751342773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,1,128,1,fp8,fp8,0,1.0082143783569335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,2,128,1,bfloat16,bfloat16,0,1.0293744087219239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,2,128,1,bfloat16,fp8,0,1.0232943534851073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,2,128,1,fp8,fp8,0,1.007151985168457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,4,128,1,bfloat16,bfloat16,0,1.0577424049377442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,4,128,1,bfloat16,fp8,0,1.015214443206787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,48,4,128,1,bfloat16,bfloat16,0,2.1509983062744142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,4,128,1,fp8,fp8,0,1.0188943862915039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,48,128,1,bfloat16,bfloat16,0,0.784335994720459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,8,128,1,bfloat16,bfloat16,0,1.1724703788757325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,48,128,1,bfloat16,fp8,0,0.6631440162658692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,8,128,1,fp8,fp8,0,1.0462896347045898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,8,128,1,bfloat16,fp8,0,1.0624832153320312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,1,128,1,bfloat16,bfloat16,0,0.5137887954711914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,48,128,1,fp8,fp8,0,0.6655248165130615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,1,128,1,bfloat16,fp8,0,0.5289535999298096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,2,128,1,bfloat16,bfloat16,0,0.5230847835540772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,2,128,1,bfloat16,fp8,0,0.5157904148101806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,2,128,1,fp8,fp8,0,0.5043791770935059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,4,128,1,bfloat16,bfloat16,0,0.5344207763671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,4,128,1,bfloat16,fp8,0,0.5326816082000733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,4,128,1,fp8,fp8,0,0.5211040019989014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,8,128,1,bfloat16,bfloat16,0,0.5556640148162841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,48,1,128,1,bfloat16,fp8,0,1.0621904373168944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,8,128,1,bfloat16,fp8,0,0.5242800235748291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,48,128,1,bfloat16,bfloat16,0,0.4054192066192627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,48,128,1,bfloat16,fp8,0,0.3426719903945923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,8,128,1,fp8,fp8,0,0.5289711952209473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,1,128,1,bfloat16,bfloat16,0,0.2678719997406006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,48,128,1,fp8,fp8,0,0.33921759128570556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,1,128,1,bfloat16,fp8,0,0.25591681003570554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,1,128,1,fp8,fp8,0,0.2634495973587036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,2,128,1,bfloat16,bfloat16,0,0.2724191904067993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,2,128,1,bfloat16,fp8,0,0.2596591949462891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,2,128,1,fp8,fp8,0,0.2628671884536743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,4,128,1,bfloat16,bfloat16,0,0.27238080501556394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,4,128,1,bfloat16,fp8,0,0.26107680797576904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,4,128,1,fp8,fp8,0,0.26833438873291016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,8,128,1,bfloat16,fp8,0,0.2699584007263184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,8,128,1,bfloat16,bfloat16,0,0.28912959098815916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,48,8,128,1,fp8,fp8,0,0.2730448007583618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,48,128,1,bfloat16,fp8,0,0.17530560493469238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,48,128,1,fp8,fp8,0,0.17912479639053344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,1,128,1,bfloat16,bfloat16,0,0.1382032036781311
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,1,128,1,bfloat16,fp8,0,0.13701119422912597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,1,128,1,fp8,fp8,0,0.13617119789123536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,2,128,1,bfloat16,bfloat16,0,0.1389248013496399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,2,128,1,bfloat16,fp8,0,0.13862240314483643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,2,128,1,fp8,fp8,0,0.13558880090713502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,4,128,1,bfloat16,bfloat16,0,0.14495999813079835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,4,128,1,bfloat16,fp8,0,0.13863840103149414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,4,128,1,fp8,fp8,0,0.13591519594192505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,8,128,1,bfloat16,bfloat16,0,0.15072799921035768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,8,128,1,bfloat16,fp8,0,0.14037439823150635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,48,128,1,bfloat16,bfloat16,0,0.11166080236434936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,48,128,1,bfloat16,fp8,0,0.09398559927940368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,48,128,1,fp8,fp8,0,0.09653279781341553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,1,128,1,bfloat16,bfloat16,0,0.0744159996509552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,1,128,1,bfloat16,fp8,0,0.07264000177383423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,1,128,1,fp8,fp8,0,0.07121599912643432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,2,128,1,bfloat16,bfloat16,0,0.07689120173454285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,2,128,1,bfloat16,fp8,0,0.07372639775276184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,2,128,1,fp8,fp8,0,0.07348639965057373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,4,128,1,bfloat16,bfloat16,0,0.07648000121116638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,4,128,1,bfloat16,fp8,0,0.07475039958953858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,4,128,1,fp8,fp8,0,0.07570080161094665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,8,128,1,bfloat16,bfloat16,0,0.08061599731445312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,8,128,1,bfloat16,fp8,0,0.07692000269889832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,48,8,128,1,fp8,fp8,0,0.07720479965209961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,48,128,1,bfloat16,bfloat16,0,0.06360160112380982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,48,128,1,bfloat16,fp8,0,0.05474079847335815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,48,128,1,fp8,fp8,0,0.055323201417922976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,1,128,1,bfloat16,fp8,0,0.03943040072917938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,1,128,1,fp8,fp8,0,0.038936001062393186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,2,128,1,bfloat16,bfloat16,0,0.04049279987812042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,2,128,1,bfloat16,fp8,0,0.0394320011138916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,48,1,128,1,fp8,fp8,0,0.5037295818328857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,2,128,1,fp8,fp8,0,0.039001598954200745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,4,128,1,bfloat16,bfloat16,0,0.040836799144744876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,4,128,1,bfloat16,fp8,0,0.03976800143718719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,8,128,1,bfloat16,bfloat16,0,0.04169920086860657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,4,128,1,fp8,fp8,0,0.03827039897441864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,8,128,1,bfloat16,fp8,0,0.04014239907264709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,8,128,1,fp8,fp8,0,0.04019519984722138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,48,128,1,bfloat16,bfloat16,0,0.2075376033782959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,48,8,128,1,fp8,fp8,0,0.14291679859161377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,1,128,1,bfloat16,bfloat16,0,2.845180892944336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,1,128,1,bfloat16,fp8,0,2.85645751953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,2,128,1,bfloat16,bfloat16,0,2.8787887573242186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,1,128,1,fp8,fp8,0,2.898423957824707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,48,1,128,1,bfloat16,bfloat16,0,0.03996959924697876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,2,128,1,bfloat16,fp8,0,2.874233627319336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,2,128,1,fp8,fp8,0,2.8824928283691404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,4,128,1,bfloat16,bfloat16,0,2.982539176940918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,4,128,1,bfloat16,fp8,0,2.9287088394165037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,4,128,1,fp8,fp8,0,2.913203239440918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,8,128,1,bfloat16,bfloat16,0,3.1448015213012694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,8,128,1,bfloat16,fp8,0,3.0565935134887696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,48,8,128,1,fp8,fp8,0,3.0756975173950196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,48,128,1,bfloat16,bfloat16,0,2.5001983642578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,1,128,1,bfloat16,bfloat16,0,1.3994864463806151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,1,128,1,bfloat16,fp8,0,1.4292336463928224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,48,128,1,bfloat16,fp8,0,2.1044912338256836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,1,128,1,fp8,fp8,0,1.4364048004150392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,2,128,1,bfloat16,bfloat16,0,1.4220591545104981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,2,128,1,bfloat16,fp8,0,1.4447888374328612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,48,128,1,fp8,fp8,0,2.1385183334350586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,2,128,1,fp8,fp8,0,1.46876802444458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,4,128,1,bfloat16,bfloat16,0,1.4660880088806152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,4,128,1,bfloat16,fp8,0,1.4633071899414063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,48,128,1,bfloat16,bfloat16,0,1.2578319549560546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,4,128,1,fp8,fp8,0,1.4615103721618652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,8,128,1,bfloat16,fp8,0,1.5355024337768555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,8,128,1,bfloat16,bfloat16,0,1.5717568397521973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,48,8,128,1,fp8,fp8,0,1.5217935562133789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,1,128,1,bfloat16,bfloat16,0,0.7088784217834473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,48,128,1,bfloat16,fp8,0,1.0725279808044434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,1,128,1,bfloat16,fp8,0,0.7204368114471436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,2,128,1,bfloat16,bfloat16,0,0.7212560176849365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,1,128,1,fp8,fp8,0,0.7188000202178955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,48,128,1,fp8,fp8,0,1.0577008247375488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,2,128,1,bfloat16,fp8,0,0.7222335815429688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,4,128,1,bfloat16,bfloat16,0,0.7447487831115722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,2,128,1,fp8,fp8,0,0.7427072048187255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,4,128,1,bfloat16,fp8,0,0.7345647811889648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,4,128,1,fp8,fp8,0,0.7416463851928711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,8,128,1,bfloat16,bfloat16,0,0.7900847911834716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,8,128,1,bfloat16,fp8,0,0.7698351860046386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,48,8,128,1,fp8,fp8,0,0.7660783767700196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,1,128,1,bfloat16,fp8,0,0.3652879953384399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,1,128,1,bfloat16,bfloat16,0,0.3596479892730713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,1,128,1,fp8,fp8,0,0.3673151969909668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,48,128,1,bfloat16,fp8,0,0.5441440105438232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,2,128,1,bfloat16,bfloat16,0,0.364247989654541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,48,128,1,bfloat16,bfloat16,0,0.6372928142547607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,2,128,1,bfloat16,fp8,0,0.36797759532928465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,2,128,1,fp8,fp8,0,0.3703743934631348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,4,128,1,bfloat16,bfloat16,0,0.3783135890960693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,4,128,1,bfloat16,fp8,0,0.3848432064056396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,8,128,1,bfloat16,bfloat16,0,0.4036303997039795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,8,128,1,bfloat16,fp8,0,0.3900576114654541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,8,128,1,fp8,fp8,0,0.3975311994552612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,48,128,1,bfloat16,fp8,0,0.27878239154815676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,48,128,1,fp8,fp8,0,0.2751760005950928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,48,128,1,bfloat16,bfloat16,0,0.32673120498657227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,1,128,1,bfloat16,bfloat16,0,0.1863487958908081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,1,128,1,bfloat16,fp8,0,0.19271839857101442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,1,128,1,fp8,fp8,0,0.1939296007156372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,2,128,1,bfloat16,fp8,0,0.1938655972480774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,2,128,1,fp8,fp8,0,0.19540640115737914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,4,128,1,bfloat16,bfloat16,0,0.19944800138473512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,4,128,1,bfloat16,fp8,0,0.19883040189743043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,4,128,1,fp8,fp8,0,0.19660799503326415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,8,128,1,bfloat16,bfloat16,0,0.21133921146392823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,8,128,1,bfloat16,fp8,0,0.20532639026641847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,48,128,1,bfloat16,bfloat16,0,0.1721232056617737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,8,128,1,fp8,fp8,0,0.20589759349822997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,48,128,1,bfloat16,fp8,0,0.14608800411224365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,1,128,1,bfloat16,bfloat16,0,0.10202720165252685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,1,128,1,bfloat16,fp8,0,0.10128639936447144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,48,128,1,fp8,fp8,0,0.14569920301437378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,1,128,1,fp8,fp8,0,0.10323840379714966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,2,128,1,bfloat16,fp8,0,0.10210560560226441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,2,128,1,fp8,fp8,0,0.10397119522094726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,4,128,1,bfloat16,bfloat16,0,0.10665279626846313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,4,128,1,bfloat16,fp8,0,0.10348479747772217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,4,128,1,fp8,fp8,0,0.10518720149993896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,8,128,1,bfloat16,bfloat16,0,0.11242400407791138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,8,128,1,bfloat16,fp8,0,0.10726560354232788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,8,128,1,fp8,fp8,0,0.10909279584884643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,48,128,1,bfloat16,bfloat16,0,0.09335839748382568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,48,128,1,bfloat16,fp8,0,0.07810559868812561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,48,128,1,fp8,fp8,0,0.07907999753952026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,1,128,1,bfloat16,bfloat16,0,0.05828160047531128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,1,128,1,bfloat16,fp8,0,0.0558031976222992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,1,128,1,fp8,fp8,0,0.056806397438049314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,2,128,1,bfloat16,bfloat16,0,0.058646398782730105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,2,128,1,bfloat16,fp8,0,0.05664960145950317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,2,128,1,fp8,fp8,0,0.057443201541900635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,4,128,1,bfloat16,bfloat16,0,0.059806400537490846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,4,128,1,bfloat16,fp8,0,0.05715519785881042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,4,128,1,fp8,fp8,0,0.05843999981880188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,8,128,1,bfloat16,bfloat16,0,0.06259040236473083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,48,128,1,fp8,fp8,0,0.5324512004852295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,8,128,1,bfloat16,fp8,0,0.05947359800338745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,48,128,1,bfloat16,fp8,0,0.045371198654174806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,48,128,1,bfloat16,bfloat16,0,0.052508801221847534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,48,8,128,1,fp8,fp8,0,0.05965920090675354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,1,128,1,bfloat16,bfloat16,0,0.03138239979743958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,48,128,1,fp8,fp8,0,0.04411520063877106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,1,128,1,bfloat16,fp8,0,0.03284319937229156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,1,128,1,fp8,fp8,0,0.03293280005455017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,2,128,1,bfloat16,bfloat16,0,0.031611201167106626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,2,128,1,bfloat16,fp8,0,0.032092800736427306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,2,128,1,fp8,fp8,0,0.03187040090560913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,4,128,1,bfloat16,bfloat16,0,0.033456000685691836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,4,128,1,bfloat16,fp8,0,0.032660800218582156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,4,128,1,fp8,fp8,0,0.03214719891548157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,48,4,128,1,fp8,fp8,0,0.3738176107406616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,8,128,1,bfloat16,fp8,0,0.03250559866428375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,8,128,1,fp8,fp8,0,0.03375039994716644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,48,128,1,bfloat16,bfloat16,0,0.03070560097694397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,48,128,1,bfloat16,fp8,0,0.02581920027732849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,1,128,1,bfloat16,bfloat16,0,0.0204352006316185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,48,128,1,fp8,fp8,0,0.02552799880504608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,1,128,1,fp8,fp8,0,0.02046079933643341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,2,128,1,bfloat16,bfloat16,0,0.02091200053691864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,2,128,1,bfloat16,fp8,0,0.020168000459671022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,2,128,1,fp8,fp8,0,0.020347200334072113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,4,128,1,bfloat16,bfloat16,0,0.021027199923992157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,4,128,1,bfloat16,fp8,0,0.020388799905776977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,4,128,1,fp8,fp8,0,0.02037599980831146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,8,128,1,bfloat16,bfloat16,0,0.021265600621700288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,8,128,1,bfloat16,fp8,0,0.022307200729846953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,8,128,1,fp8,fp8,0,0.022065599262714387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,48,2,128,1,bfloat16,bfloat16,0,0.1898576021194458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,1,128,1,bfloat16,bfloat16,0,1.1103232383728028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,1,128,1,bfloat16,fp8,0,1.1771583557128906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,48,2,128,1,bfloat16,bfloat16,0,0.10327839851379395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,1,128,1,fp8,fp8,0,1.1767552375793457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,2,128,1,bfloat16,bfloat16,0,1.1407072067260742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,2,128,1,bfloat16,fp8,0,1.1854304313659667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,2,128,1,fp8,fp8,0,1.1849360466003418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,4,128,1,bfloat16,bfloat16,0,1.1816191673278809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,4,128,1,bfloat16,fp8,0,1.2088848114013673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,4,128,1,fp8,fp8,0,1.217625617980957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,48,8,128,1,bfloat16,bfloat16,0,0.033687999844551085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,48,1,128,1,bfloat16,fp8,0,0.020812800526618956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,1,128,1,bfloat16,bfloat16,0,0.5626272201538086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,8,128,1,bfloat16,bfloat16,0,1.2719792366027831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,48,128,1,bfloat16,fp8,0,0.9025888442993164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,48,128,1,bfloat16,bfloat16,0,1.155128002166748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,8,128,1,bfloat16,fp8,0,1.2649040222167969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,1,128,1,bfloat16,fp8,0,0.5949679851531983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,1,128,1,fp8,fp8,0,0.594916820526123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,2,128,1,bfloat16,bfloat16,0,0.5733376026153565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,2,128,1,fp8,fp8,0,0.5983407974243165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,4,128,1,bfloat16,bfloat16,0,0.5959824085235595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,4,128,1,bfloat16,fp8,0,0.6091311931610107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,4,128,1,fp8,fp8,0,0.6120560169219971
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,8,128,1,bfloat16,bfloat16,0,0.6391056060791016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,8,128,1,bfloat16,fp8,0,0.6393152236938476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,8,128,1,fp8,fp8,0,0.6359407901763916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,48,128,1,bfloat16,bfloat16,0,0.5854928016662597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,48,128,1,bfloat16,fp8,0,0.45506720542907714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,1,128,1,bfloat16,bfloat16,0,0.2853791952133179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,1,128,1,fp8,fp8,0,0.3008512020111084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,1,128,1,bfloat16,fp8,0,0.30354559421539307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,48,128,1,fp8,fp8,0,0.45750880241394043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,2,128,1,bfloat16,bfloat16,0,0.2917488098144531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,2,128,1,bfloat16,fp8,0,0.3039504051208496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,2,128,1,fp8,fp8,0,0.30605120658874513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,4,128,1,bfloat16,bfloat16,0,0.30494399070739747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,4,128,1,bfloat16,fp8,0,0.3089823961257935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,4,128,1,fp8,fp8,0,0.3101232051849365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,8,128,1,bfloat16,bfloat16,0,0.32792479991912843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,48,128,1,fp8,fp8,0,0.8994992256164551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,48,8,128,1,fp8,fp8,0,1.265113639831543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,8,128,1,fp8,fp8,0,0.3234096050262451
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,48,128,1,bfloat16,bfloat16,0,0.30074880123138426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,48,128,1,bfloat16,fp8,0,0.23658399581909179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,1,128,1,bfloat16,bfloat16,0,0.15052160024642944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,48,128,1,fp8,fp8,0,0.23646080493927002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,1,128,1,bfloat16,fp8,0,0.15605440139770507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,2,128,1,bfloat16,bfloat16,0,0.15525280237197875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,1,128,1,fp8,fp8,0,0.15902559757232665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,2,128,1,bfloat16,fp8,0,0.15880320072174073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,2,128,1,fp8,fp8,0,0.1591312050819397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,4,128,1,bfloat16,bfloat16,0,0.15878080129623412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,48,2,128,1,bfloat16,fp8,0,0.5968624114990234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,4,128,1,bfloat16,fp8,0,0.16320960521697997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,4,128,1,fp8,fp8,0,0.16352800130844117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,8,128,1,bfloat16,bfloat16,0,0.1712015986442566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,48,128,1,bfloat16,bfloat16,0,0.15839040279388428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,48,128,1,bfloat16,fp8,0,0.12456799745559692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,48,128,1,fp8,fp8,0,0.12608319520950317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,1,128,1,bfloat16,fp8,0,0.08504480123519897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,2,128,1,bfloat16,bfloat16,0,0.08327680230140685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,1,128,1,fp8,fp8,0,0.08410720229148864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,2,128,1,bfloat16,fp8,0,0.08673920035362244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,2,128,1,fp8,fp8,0,0.08616799712181092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,4,128,1,bfloat16,bfloat16,0,0.08589919805526733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,4,128,1,bfloat16,fp8,0,0.08650879859924317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,4,128,1,fp8,fp8,0,0.08855040073394775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,8,128,1,bfloat16,bfloat16,0,0.09172160029411316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,8,128,1,fp8,fp8,0,0.09016799926757812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,48,128,1,bfloat16,bfloat16,0,0.08699039816856384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,48,128,1,bfloat16,fp8,0,0.0693615972995758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,48,128,1,fp8,fp8,0,0.06851999759674073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,1,128,1,bfloat16,fp8,0,0.04712800085544586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,1,128,1,fp8,fp8,0,0.04631519913673401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,2,128,1,bfloat16,bfloat16,0,0.04805760085582733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,2,128,1,bfloat16,fp8,0,0.04704799950122833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,2,128,1,fp8,fp8,0,0.04799999892711639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,4,128,1,bfloat16,bfloat16,0,0.050307202339172366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,4,128,1,bfloat16,fp8,0,0.04894559979438782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,4,128,1,fp8,fp8,0,0.04941760003566742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,8,128,1,bfloat16,bfloat16,0,0.052476799488067626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,8,128,1,bfloat16,fp8,0,0.05209760069847107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,8,128,1,fp8,fp8,0,0.05194720029830933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,48,128,1,bfloat16,bfloat16,0,0.04912639856338501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,48,128,1,bfloat16,fp8,0,0.039854401350021364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,48,128,1,fp8,fp8,0,0.04009920060634613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,1,128,1,bfloat16,bfloat16,0,0.02552480101585388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,1,128,1,bfloat16,fp8,0,0.02826719880104065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,1,128,1,fp8,fp8,0,0.0271807998418808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,2,128,1,bfloat16,bfloat16,0,0.026086398959159852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,2,128,1,bfloat16,fp8,0,0.02732959985733032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,2,128,1,fp8,fp8,0,0.02773439884185791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,4,128,1,bfloat16,bfloat16,0,0.026208001375198364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,4,128,1,bfloat16,fp8,0,0.02741760015487671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,4,128,1,fp8,fp8,0,0.027326399087905885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,48,8,128,1,bfloat16,fp8,0,0.32698400020599366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,8,128,1,bfloat16,bfloat16,0,0.027641600370407103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,8,128,1,fp8,fp8,0,0.02789919972419739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,48,8,128,1,bfloat16,fp8,0,0.027735999226570128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,48,128,1,bfloat16,bfloat16,0,0.02786400020122528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,48,128,1,bfloat16,fp8,0,0.02228959947824478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,48,128,1,fp8,fp8,0,0.022668799757957457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,1,128,1,bfloat16,bfloat16,0,0.01896799951791763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,1,128,1,bfloat16,fp8,0,0.018721599876880646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,1,128,1,fp8,fp8,0,0.018376000225543976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,2,128,1,bfloat16,bfloat16,0,0.0191648006439209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,2,128,1,bfloat16,fp8,0,0.018438400328159334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,2,128,1,fp8,fp8,0,0.018671999871730804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,4,128,1,bfloat16,bfloat16,0,0.0189423993229866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,4,128,1,bfloat16,fp8,0,0.018529599905014037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,4,128,1,fp8,fp8,0,0.018566399812698364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,8,128,1,bfloat16,bfloat16,0,0.01961120069026947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,8,128,1,bfloat16,fp8,0,0.02038239985704422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,8,128,1,fp8,fp8,0,0.17077120542526245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,48,8,128,1,fp8,fp8,0,0.020478400588035583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,48,128,1,bfloat16,bfloat16,0,0.016484799981117248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,1,128,1,bfloat16,bfloat16,0,0.08339840173721313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,48,128,1,fp8,fp8,0,0.01626240015029907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,1,128,1,bfloat16,bfloat16,0,0.014374400675296783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,1,128,1,bfloat16,fp8,0,0.015300799906253815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,2,128,1,bfloat16,bfloat16,0,0.014428800344467163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,2,128,1,bfloat16,fp8,0,0.015230399370193482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,2,128,1,fp8,fp8,0,0.015043200552463531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,4,128,1,bfloat16,bfloat16,0,0.014703999459743499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,4,128,1,bfloat16,fp8,0,0.01512800008058548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,4,128,1,fp8,fp8,0,0.01528480052947998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,48,8,128,1,bfloat16,fp8,0,0.09046239852905273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,8,128,1,bfloat16,fp8,0,0.015615999698638916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,8,128,1,fp8,fp8,0,0.015440000593662262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,48,1,128,1,bfloat16,bfloat16,0,0.04809440076351166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,1,128,1,bfloat16,bfloat16,0,0.5398320198059082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,1,128,1,bfloat16,fp8,0,0.5825808048248291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,1,128,1,fp8,fp8,0,0.582153606414795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,2,128,1,bfloat16,bfloat16,0,0.5482431888580322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,2,128,1,bfloat16,fp8,0,0.5922912120819092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,2,128,1,fp8,fp8,0,0.5919184207916259
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,4,128,1,bfloat16,bfloat16,0,0.5682464122772217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,4,128,1,bfloat16,fp8,0,0.613593578338623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,4,128,1,fp8,fp8,0,0.6137824058532715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,48,8,128,1,bfloat16,fp8,0,0.16982560157775878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,48,128,1,bfloat16,fp8,0,0.01648640036582947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,8,128,1,bfloat16,fp8,0,0.644755220413208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,1,128,1,fp8,fp8,0,0.01523679941892624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,48,8,128,1,bfloat16,bfloat16,0,0.014871999621391296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,1,128,1,bfloat16,bfloat16,0,0.27743360996246336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,48,128,1,bfloat16,bfloat16,0,0.5767695903778076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,48,128,1,bfloat16,fp8,0,0.4771455764770508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,48,128,1,fp8,fp8,0,0.47608962059021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,1,128,1,bfloat16,fp8,0,0.2972192049026489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,1,128,1,fp8,fp8,0,0.2957040071487427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,2,128,1,bfloat16,bfloat16,0,0.28268959522247317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,2,128,1,bfloat16,fp8,0,0.3019903898239136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,2,128,1,fp8,fp8,0,0.3044127941131592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,4,128,1,bfloat16,bfloat16,0,0.29217441082000734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,4,128,1,bfloat16,fp8,0,0.3130575895309448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,4,128,1,fp8,fp8,0,0.31547999382019043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,8,128,1,bfloat16,bfloat16,0,0.311028790473938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,8,128,1,bfloat16,fp8,0,0.3301680088043213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,48,8,128,1,fp8,fp8,0,0.3298192024230957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,48,128,1,bfloat16,fp8,0,0.24385600090026854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,1,128,1,bfloat16,bfloat16,0,0.14519519805908204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,48,128,1,fp8,fp8,0,0.24508318901062012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,1,128,1,fp8,fp8,0,0.15337120294570922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,2,128,1,bfloat16,bfloat16,0,0.14647200107574462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,2,128,1,bfloat16,fp8,0,0.15593440532684327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,2,128,1,fp8,fp8,0,0.1560976028442383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,4,128,1,bfloat16,bfloat16,0,0.15237439870834352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,4,128,1,bfloat16,fp8,0,0.16295360326766967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,4,128,1,fp8,fp8,0,0.16137280464172363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,8,128,1,bfloat16,fp8,0,0.1701632022857666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,8,128,1,fp8,fp8,0,0.1726431965827942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,48,128,1,bfloat16,bfloat16,0,0.15885119438171386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,48,128,1,bfloat16,fp8,0,0.12749919891357422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,48,128,1,fp8,fp8,0,0.12757279872894287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,1,128,1,bfloat16,bfloat16,0,0.08057119846343994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,1,128,1,bfloat16,fp8,0,0.08293439745903015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,1,128,1,fp8,fp8,0,0.08374080061912537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,2,128,1,bfloat16,fp8,0,0.08507680296897888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,2,128,1,fp8,fp8,0,0.08476160168647766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,8,128,1,bfloat16,bfloat16,0,0.6066383838653564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,4,128,1,bfloat16,bfloat16,0,0.0810591995716095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,4,128,1,bfloat16,fp8,0,0.08640639781951905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,8,128,1,bfloat16,bfloat16,0,0.08689280152320862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,4,128,1,fp8,fp8,0,0.0860047996044159
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,48,8,128,1,fp8,fp8,0,0.6450367927551269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,8,128,1,fp8,fp8,0,0.08945119976997376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,48,128,1,bfloat16,bfloat16,0,0.0869055986404419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,48,128,1,bfloat16,fp8,0,0.06740319728851318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,48,128,1,fp8,fp8,0,0.06701120138168334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,1,128,1,bfloat16,bfloat16,0,0.04555040001869202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,1,128,1,bfloat16,fp8,0,0.043639999628067014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,1,128,1,fp8,fp8,0,0.04395999908447266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,2,128,1,bfloat16,bfloat16,0,0.04588479995727539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,2,128,1,bfloat16,fp8,0,0.04431839883327484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,2,128,1,fp8,fp8,0,0.04445120096206665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,4,128,1,bfloat16,bfloat16,0,0.04636160135269165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,4,128,1,bfloat16,fp8,0,0.045840001106262206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,4,128,1,fp8,fp8,0,0.04613440036773682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,8,128,1,bfloat16,bfloat16,0,0.04978399872779846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,8,128,1,fp8,fp8,0,0.04859519898891449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,48,128,1,bfloat16,bfloat16,0,0.04695200026035309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,48,128,1,bfloat16,fp8,0,0.03576320111751556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,48,128,1,fp8,fp8,0,0.035446399450302125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,1,128,1,bfloat16,bfloat16,0,0.02282399982213974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,1,128,1,bfloat16,fp8,0,0.024400000274181367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,1,128,1,bfloat16,fp8,0,0.1522320032119751
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,1,128,1,fp8,fp8,0,0.024216000735759736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,2,128,1,bfloat16,bfloat16,0,0.022859199345111846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,2,128,1,bfloat16,fp8,0,0.02423039972782135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,2,128,1,fp8,fp8,0,0.023945599794387817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,4,128,1,bfloat16,bfloat16,0,0.023537600040435792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,4,128,1,bfloat16,fp8,0,0.024031999707221984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,4,128,1,fp8,fp8,0,0.024371199309825897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,8,128,1,bfloat16,bfloat16,0,0.02473759949207306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,8,128,1,bfloat16,fp8,0,0.02465600073337555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,8,128,1,bfloat16,bfloat16,0,0.16284639835357667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,48,8,128,1,fp8,fp8,0,0.02438880056142807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,48,128,1,bfloat16,bfloat16,0,0.026228800415992737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,48,128,1,bfloat16,fp8,0,0.020688000321388244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,48,128,1,fp8,fp8,0,0.020046399533748628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,1,128,1,bfloat16,bfloat16,0,0.017603200674057008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,1,128,1,bfloat16,fp8,0,0.01880960017442703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,1,128,1,fp8,fp8,0,0.018302400410175324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,2,128,1,bfloat16,bfloat16,0,0.017643199861049653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,2,128,1,bfloat16,fp8,0,0.018785600364208222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,2,128,1,fp8,fp8,0,0.018196800351142885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,4,128,1,bfloat16,fp8,0,0.01884319931268692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,4,128,1,fp8,fp8,0,0.018464000523090364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,8,128,1,bfloat16,bfloat16,0,0.018460799753665925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,2,128,1,bfloat16,bfloat16,0,0.079230397939682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,8,128,1,bfloat16,fp8,0,0.018644799292087556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,8,128,1,fp8,fp8,0,0.018825599551200868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,48,128,1,bfloat16,bfloat16,0,0.01523360013961792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,48,128,1,fp8,fp8,0,0.014500799775123595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,1,128,1,bfloat16,bfloat16,0,0.01282079964876175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,1,128,1,bfloat16,fp8,0,0.013393600285053254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,1,128,1,fp8,fp8,0,0.013492800295352936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,2,128,1,bfloat16,bfloat16,0,0.012768000364303589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,2,128,1,bfloat16,fp8,0,0.01356319934129715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,2,128,1,fp8,fp8,0,0.013612799346446991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,4,128,1,bfloat16,bfloat16,0,0.013339200615882873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,48,8,128,1,bfloat16,fp8,0,0.08954880237579346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,4,128,1,bfloat16,fp8,0,0.01343040019273758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,4,128,1,fp8,fp8,0,0.013627199828624726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,8,128,1,bfloat16,bfloat16,0,0.013355199992656708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,8,128,1,fp8,fp8,0,0.013859200477600097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,48,128,1,bfloat16,bfloat16,0,0.014097599685192109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,48,128,1,bfloat16,fp8,0,0.013995200395584106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,48,128,1,fp8,fp8,0,0.013740800321102142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,1,128,1,bfloat16,bfloat16,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,1,128,1,bfloat16,fp8,0,0.013016000390052795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,1,128,1,fp8,fp8,0,0.013167999684810638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,2,128,1,bfloat16,bfloat16,0,0.012428800016641617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,2,128,1,bfloat16,fp8,0,0.013142399489879608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,2,128,1,fp8,fp8,0,0.01308639943599701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,4,128,1,bfloat16,bfloat16,0,0.012940800189971924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,4,128,1,bfloat16,fp8,0,0.013023999333381654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,4,128,1,fp8,fp8,0,0.013335999846458436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,8,128,1,bfloat16,bfloat16,0,0.012848000228404998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,48,8,128,1,bfloat16,fp8,0,0.04796960055828094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,8,128,1,fp8,fp8,0,0.013211199641227722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,1,128,1,bfloat16,bfloat16,0,0.2774208068847656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,1,128,1,bfloat16,fp8,0,0.3017215967178345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,48,48,128,1,bfloat16,bfloat16,0,0.2997136116027832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,2,128,1,bfloat16,bfloat16,0,0.2813375949859619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,2,128,1,fp8,fp8,0,0.30386240482330323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,4,128,1,bfloat16,bfloat16,0,0.29181599617004395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,48,4,128,1,bfloat16,bfloat16,0,0.017892800271511078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,4,128,1,bfloat16,fp8,0,0.31529760360717773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,4,128,1,fp8,fp8,0,0.3147680044174194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,8,128,1,bfloat16,bfloat16,0,0.3133311986923218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,8,128,1,bfloat16,fp8,0,0.3314671993255615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,8,128,1,bfloat16,fp8,0,0.01361120045185089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,8,128,1,fp8,fp8,0,0.3308720111846924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,48,128,1,bfloat16,bfloat16,0,0.36526238918304443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,48,128,1,bfloat16,fp8,0,0.348140811920166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,1,128,1,bfloat16,bfloat16,0,0.14479199647903443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,48,128,1,fp8,fp8,0,0.34782240390777586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,1,128,1,bfloat16,fp8,0,0.155076801776886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,1,128,1,fp8,fp8,0,0.15521440505981446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,48,8,128,1,bfloat16,fp8,0,0.013596799969673157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,2,128,1,bfloat16,fp8,0,0.15570399761199952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,2,128,1,fp8,fp8,0,0.1583359956741333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,4,128,1,bfloat16,bfloat16,0,0.153056001663208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,4,128,1,bfloat16,fp8,0,0.16459200382232667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,4,128,1,fp8,fp8,0,0.16034239530563354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,8,128,1,bfloat16,bfloat16,0,0.16399680376052855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,8,128,1,bfloat16,fp8,0,0.17275840044021606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,8,128,1,fp8,fp8,0,0.17419519424438476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,48,128,1,bfloat16,bfloat16,0,0.19022719860076903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,48,128,1,bfloat16,fp8,0,0.18259199857711791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,1,128,1,fp8,fp8,0,0.2999855995178223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,1,128,1,bfloat16,bfloat16,0,0.07746880054473877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,1,128,1,bfloat16,fp8,0,0.0844223976135254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,48,128,1,fp8,fp8,0,0.1816223978996277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,1,128,1,fp8,fp8,0,0.08420640230178833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,2,128,1,bfloat16,bfloat16,0,0.07872639894485474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,2,128,1,bfloat16,fp8,0,0.08603360056877137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,2,128,1,fp8,fp8,0,0.08468319773674012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,4,128,1,bfloat16,bfloat16,0,0.08307520151138306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,4,128,1,bfloat16,fp8,0,0.08731200098991394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,4,128,1,fp8,fp8,0,0.08758720159530639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,48,2,128,1,bfloat16,fp8,0,0.30576159954071047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,8,128,1,bfloat16,bfloat16,0,0.08727040290832519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,8,128,1,bfloat16,fp8,0,0.09114720225334168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,48,8,128,1,fp8,fp8,0,0.08981919884681702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,48,128,1,bfloat16,bfloat16,0,0.10132800340652466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,1,128,1,bfloat16,bfloat16,0,0.046142399311065674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,48,128,1,fp8,fp8,0,0.09475839734077454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,1,128,1,bfloat16,fp8,0,0.04442720115184784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,1,128,1,fp8,fp8,0,0.04359839856624603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,2,128,1,bfloat16,bfloat16,0,0.04687039852142334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,2,128,1,bfloat16,fp8,0,0.04525760114192963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,2,128,1,fp8,fp8,0,0.045638400316238406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,4,128,1,bfloat16,bfloat16,0,0.046891200542449954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,4,128,1,bfloat16,fp8,0,0.04687199890613556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,4,128,1,fp8,fp8,0,0.04704160094261169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,8,128,1,bfloat16,bfloat16,0,0.049807998538017276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,8,128,1,bfloat16,fp8,0,0.04815680086612702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,8,128,1,fp8,fp8,0,0.04896480143070221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,48,128,1,bfloat16,fp8,0,0.04855999946594238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,48,128,1,fp8,fp8,0,0.04817120134830475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,1,128,1,bfloat16,bfloat16,0,0.022856000065803527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,48,48,128,1,bfloat16,fp8,0,0.014427199959754944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,1,128,1,bfloat16,fp8,0,0.02456959933042526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,1,128,1,fp8,fp8,0,0.024372799694538115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,2,128,1,bfloat16,bfloat16,0,0.022899200022220612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,2,128,1,fp8,fp8,0,0.024532799422740937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,4,128,1,bfloat16,bfloat16,0,0.0234607994556427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,4,128,1,bfloat16,fp8,0,0.024268800020217897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,4,128,1,fp8,fp8,0,0.024596799910068513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,8,128,1,bfloat16,bfloat16,0,0.02443680018186569
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,8,128,1,fp8,fp8,0,0.024956800043582916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,48,128,1,bfloat16,bfloat16,0,0.02818560004234314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,48,128,1,bfloat16,fp8,0,0.0247296005487442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,48,128,1,fp8,fp8,0,0.024187199771404266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,1,128,1,bfloat16,bfloat16,0,0.01778720021247864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,1,128,1,bfloat16,fp8,0,0.018671999871730804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,1,128,1,fp8,fp8,0,0.019038400053977965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,2,128,1,bfloat16,bfloat16,0,0.01746080070734024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,2,128,1,bfloat16,fp8,0,0.018911999464035035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,2,128,1,fp8,fp8,0,0.018670399487018586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,4,128,1,bfloat16,bfloat16,0,0.017961600422859193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,4,128,1,bfloat16,fp8,0,0.01871040016412735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,4,128,1,fp8,fp8,0,0.019047999382019044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,8,128,1,bfloat16,bfloat16,0,0.017833599448204042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,8,128,1,bfloat16,fp8,0,0.019176000356674196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,48,8,128,1,fp8,fp8,0,0.018751999735832213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,48,128,1,bfloat16,bfloat16,0,0.01828960031270981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,48,128,1,bfloat16,fp8,0,0.018385599553585052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,48,128,1,fp8,fp8,0,0.0185248002409935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,1,128,1,bfloat16,bfloat16,0,0.012644800543785095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,1,128,1,bfloat16,fp8,0,0.013547199964523315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,1,128,1,fp8,fp8,0,0.013473600149154663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,2,128,1,bfloat16,bfloat16,0,0.012851199507713318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,2,128,1,bfloat16,fp8,0,0.013329599797725678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,2,128,1,fp8,fp8,0,0.013460800051689148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,4,128,1,bfloat16,bfloat16,0,0.012772800028324127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,4,128,1,bfloat16,fp8,0,0.013624000549316406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,4,128,1,fp8,fp8,0,0.013643200695514678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,8,128,1,bfloat16,bfloat16,0,0.013089600205421447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,8,128,1,fp8,fp8,0,0.013715200126171112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,48,128,1,bfloat16,bfloat16,0,0.013548800349235534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,48,128,1,bfloat16,fp8,0,0.01372160017490387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,48,128,1,fp8,fp8,0,0.013729600608348847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,1,128,1,bfloat16,bfloat16,0,0.01268479973077774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,1,128,1,bfloat16,fp8,0,0.01343040019273758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,1,128,1,fp8,fp8,0,0.013223999738693237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,2,128,1,bfloat16,bfloat16,0,0.012604799866676331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,48,48,128,1,bfloat16,fp8,0,0.09526079893112183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,2,128,1,bfloat16,fp8,0,0.013211199641227722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,2,128,1,fp8,fp8,0,0.013225600123405457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,4,128,1,bfloat16,bfloat16,0,0.012751999497413635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,4,128,1,bfloat16,fp8,0,0.013419200479984284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,4,128,1,fp8,fp8,0,0.013275200128555298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,8,128,1,bfloat16,bfloat16,0,0.01276959925889969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,8,128,1,bfloat16,fp8,0,0.01340479999780655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,48,8,128,1,fp8,fp8,0,0.01329759955406189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,48,128,1,bfloat16,bfloat16,0,0.01311199963092804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,48,128,1,bfloat16,bfloat16,0,0.05599200129508972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,48,128,1,bfloat16,fp8,0,0.013571199774742127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,48,128,1,fp8,fp8,0,0.013550400733947754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,1,128,1,bfloat16,fp8,0,0.01308320015668869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,1,128,1,fp8,fp8,0,0.013075199723243714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,2,128,1,bfloat16,bfloat16,0,0.01249919980764389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,2,128,1,bfloat16,fp8,0,0.012999999523162841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,2,128,1,fp8,fp8,0,0.013193599879741669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,4,128,1,bfloat16,bfloat16,0,0.012503999471664428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,4,128,1,bfloat16,fp8,0,0.013155199587345123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,8,128,1,bfloat16,fp8,0,0.024600000679492952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,4,128,1,fp8,fp8,0,0.01310880035161972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,8,128,1,bfloat16,bfloat16,0,0.012507200241088867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,8,128,1,bfloat16,fp8,0,0.01318880021572113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,8,128,1,fp8,fp8,0,0.01302880048751831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,1,128,1,bfloat16,bfloat16,0,0.14433120489120482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,1,128,1,bfloat16,fp8,0,0.15780160427093506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,1,128,1,fp8,fp8,0,0.15330079793930054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,2,128,1,bfloat16,bfloat16,0,0.1473888039588928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,2,128,1,bfloat16,fp8,0,0.15704480409622193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,2,128,1,fp8,fp8,0,0.16003680229187012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,48,2,128,1,bfloat16,bfloat16,0,0.14759680032730102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,4,128,1,bfloat16,bfloat16,0,0.15350079536437988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,4,128,1,bfloat16,fp8,0,0.16470400094985962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,4,128,1,fp8,fp8,0,0.1612112045288086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,8,128,1,bfloat16,bfloat16,0,0.19476159811019897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,8,128,1,bfloat16,fp8,0,0.19767999649047852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,48,8,128,1,fp8,fp8,0,0.19715839624404907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,48,8,128,1,bfloat16,fp8,0,0.013604800403118133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,48,128,1,bfloat16,bfloat16,0,0.27808959484100343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,1,128,1,bfloat16,bfloat16,0,0.07885119915008545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,48,128,1,bfloat16,fp8,0,0.28286399841308596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,48,128,1,fp8,fp8,0,0.28223841190338134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,1,128,1,bfloat16,fp8,0,0.08510720133781433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,2,128,1,bfloat16,bfloat16,0,0.07929440140724182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,2,128,1,bfloat16,fp8,0,0.08633919954299926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,2,128,1,fp8,fp8,0,0.0868943989276886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,4,128,1,bfloat16,bfloat16,0,0.08113920092582702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,4,128,1,bfloat16,fp8,0,0.08807839751243592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,4,128,1,fp8,fp8,0,0.08816480040550231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,8,128,1,bfloat16,bfloat16,0,0.10251040458679199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,8,128,1,bfloat16,fp8,0,0.10623680353164673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,8,128,1,fp8,fp8,0,0.10614559650421143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,48,128,1,bfloat16,bfloat16,0,0.14550559520721434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,48,128,1,bfloat16,fp8,0,0.14800000190734863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,48,128,1,fp8,fp8,0,0.14730240106582643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,1,128,1,bfloat16,fp8,0,0.044843199849128726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,1,128,1,fp8,fp8,0,0.04463199973106384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,2,128,1,bfloat16,bfloat16,0,0.04712960124015808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,2,128,1,bfloat16,fp8,0,0.04560160040855408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,2,128,1,fp8,fp8,0,0.045332801342010495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,48,2,128,1,bfloat16,fp8,0,0.024451200664043427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,4,128,1,bfloat16,bfloat16,0,0.047737601399421695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,4,128,1,bfloat16,fp8,0,0.04819679856300354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,4,128,1,fp8,fp8,0,0.04768800139427185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,8,128,1,bfloat16,bfloat16,0,0.058387202024459836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,8,128,1,bfloat16,fp8,0,0.05819839835166931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,8,128,1,fp8,fp8,0,0.058011198043823244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,48,128,1,bfloat16,fp8,0,0.07654079794883728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,1,128,1,bfloat16,bfloat16,0,0.023403200507164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,48,128,1,bfloat16,bfloat16,0,0.07909759879112244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,48,128,1,fp8,fp8,0,0.0772000014781952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,1,128,1,bfloat16,fp8,0,0.024710400402545928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,1,128,1,fp8,fp8,0,0.024668799340724946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,2,128,1,bfloat16,bfloat16,0,0.023291200399398804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,2,128,1,bfloat16,fp8,0,0.024806399643421174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,2,128,1,fp8,fp8,0,0.02468000054359436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,4,128,1,bfloat16,bfloat16,0,0.0238319993019104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,4,128,1,bfloat16,fp8,0,0.02478879988193512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,4,128,1,fp8,fp8,0,0.024864000082015992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,8,128,1,bfloat16,bfloat16,0,0.02858879864215851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,8,128,1,bfloat16,fp8,0,0.029467201232910155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,48,8,128,1,fp8,fp8,0,0.029447999596595765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,48,128,1,bfloat16,bfloat16,0,0.040729600191116336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,48,128,1,fp8,fp8,0,0.0384799987077713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,1,128,1,bfloat16,bfloat16,0,0.0174575999379158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,1,128,1,bfloat16,fp8,0,0.018775999546051025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,1,128,1,fp8,fp8,0,0.018764799833297728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,2,128,1,bfloat16,bfloat16,0,0.017531199753284453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,2,128,1,bfloat16,fp8,0,0.018699200451374055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,2,128,1,fp8,fp8,0,0.018822400271892546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,4,128,1,bfloat16,bfloat16,0,0.017643199861049653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,4,128,1,bfloat16,fp8,0,0.018670399487018586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,4,128,1,fp8,fp8,0,0.018777599930763243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,8,128,1,bfloat16,bfloat16,0,0.017560000717639922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,8,128,1,bfloat16,fp8,0,0.018638400733470915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,8,128,1,fp8,fp8,0,0.018785600364208222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,48,128,1,bfloat16,bfloat16,0,0.02234400063753128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,48,128,1,bfloat16,fp8,0,0.023307199776172637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,48,128,1,fp8,fp8,0,0.023080000281333925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,1,128,1,bfloat16,fp8,0,0.013665600121021271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,1,128,1,fp8,fp8,0,0.013518400490283966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,2,128,1,bfloat16,bfloat16,0,0.013089600205421447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,2,128,1,bfloat16,fp8,0,0.013643200695514678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,2,128,1,fp8,fp8,0,0.013715200126171112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,4,128,1,bfloat16,bfloat16,0,0.01300320029258728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,4,128,1,bfloat16,fp8,0,0.013655999302864074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,4,128,1,fp8,fp8,0,0.013736000657081604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,8,128,1,bfloat16,fp8,0,0.013620799779891968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,8,128,1,fp8,fp8,0,0.013775999844074249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,48,1,128,1,fp8,fp8,0,0.08599039912223816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,48,128,1,bfloat16,bfloat16,0,0.017263999581336974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,48,128,1,bfloat16,fp8,0,0.017923200130462648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,48,128,1,fp8,fp8,0,0.017793600261211396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,1,128,1,bfloat16,bfloat16,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,1,128,1,bfloat16,fp8,0,0.01313440054655075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,1,128,1,fp8,fp8,0,0.01329600065946579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,2,128,1,bfloat16,bfloat16,0,0.012488000094890594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,2,128,1,bfloat16,fp8,0,0.013225600123405457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,2,128,1,fp8,fp8,0,0.01308799982070923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,4,128,1,bfloat16,bfloat16,0,0.012620800733566284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,4,128,1,bfloat16,fp8,0,0.013339200615882873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,4,128,1,fp8,fp8,0,0.01321599930524826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,8,128,1,bfloat16,bfloat16,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,48,1,128,1,bfloat16,bfloat16,0,0.04710240066051483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,8,128,1,bfloat16,fp8,0,0.01326880007982254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,48,8,128,1,fp8,fp8,0,0.013230399787425995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,48,128,1,bfloat16,bfloat16,0,0.013307200372219085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,48,128,1,bfloat16,fp8,0,0.013424000144004822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,48,128,1,fp8,fp8,0,0.01340319961309433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,48,1,128,1,bfloat16,bfloat16,0,0.012375999987125397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,1,128,1,bfloat16,bfloat16,0,0.012455999851226807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,1,128,1,bfloat16,fp8,0,0.013179199397563934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,1,128,1,fp8,fp8,0,0.013031999766826629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,2,128,1,bfloat16,bfloat16,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,2,128,1,bfloat16,fp8,0,0.013116799294948578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,2,128,1,fp8,fp8,0,0.013022400438785553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,4,128,1,fp8,fp8,0,0.013084800541400909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,4,128,1,bfloat16,fp8,0,0.013118399679660797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,8,128,1,bfloat16,bfloat16,0,0.012465599924325943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,8,128,1,bfloat16,fp8,0,0.01297439932823181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,8,128,1,fp8,fp8,0,0.013284799456596375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,48,128,1,bfloat16,bfloat16,0,0.013199999928474426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,48,128,1,bfloat16,fp8,0,0.013214400410652161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,48,128,1,fp8,fp8,0,0.013305599987506866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,1,128,1,bfloat16,bfloat16,0,0.012251199781894683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,1,128,1,bfloat16,fp8,0,0.012647999823093415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,1,128,1,fp8,fp8,0,0.01279519945383072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,2,128,1,bfloat16,bfloat16,0,0.012382400035858155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,2,128,1,bfloat16,fp8,0,0.012667199969291687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,2,128,1,fp8,fp8,0,0.012800000607967377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,4,128,1,bfloat16,bfloat16,0,0.012240000069141388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,4,128,1,bfloat16,fp8,0,0.01295199990272522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,4,128,1,fp8,fp8,0,0.012993599474430084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,8,128,1,bfloat16,bfloat16,0,0.012454400211572647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,8,128,1,bfloat16,fp8,0,0.012902399897575379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,48,48,128,1,bfloat16,fp8,0,0.03804959952831268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,1,128,1,bfloat16,bfloat16,0,0.0793727993965149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,1,128,1,bfloat16,fp8,0,0.08604959845542907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,1,128,1,fp8,fp8,0,0.08617119789123535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,2,128,1,bfloat16,bfloat16,0,0.08062400221824646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,2,128,1,bfloat16,fp8,0,0.0872879981994629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,2,128,1,fp8,fp8,0,0.0875440001487732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,4,128,1,bfloat16,bfloat16,0,0.09848960041999817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,4,128,1,bfloat16,fp8,0,0.10448479652404785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,4,128,1,fp8,fp8,0,0.10431040525436401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,1,128,1,bfloat16,bfloat16,0,0.01297599971294403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,8,128,1,bfloat16,bfloat16,0,0.10513759851455688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,8,128,1,bfloat16,fp8,0,0.10833439826965333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,48,8,128,1,fp8,fp8,0,0.10837279558181763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,1,128,1,bfloat16,bfloat16,0,0.04657120108604431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,48,128,1,bfloat16,fp8,0,0.25686399936676024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,48,128,1,bfloat16,bfloat16,0,0.237280011177063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,48,8,128,1,bfloat16,bfloat16,0,0.013227200508117676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,48,128,1,fp8,fp8,0,0.2568975925445557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,1,128,1,bfloat16,fp8,0,0.04600160121917725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,1,128,1,fp8,fp8,0,0.04615199863910675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,2,128,1,bfloat16,bfloat16,0,0.047619199752807616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,2,128,1,fp8,fp8,0,0.0468095988035202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,2,128,1,bfloat16,fp8,0,0.046953600645065305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,4,128,1,bfloat16,bfloat16,0,0.05601760149002075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,4,128,1,fp8,fp8,0,0.0572160005569458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,4,128,1,bfloat16,fp8,0,0.05556960105895996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,8,128,1,bfloat16,fp8,0,0.060099202394485476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,48,128,1,bfloat16,bfloat16,0,0.12552640438079835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,48,128,1,bfloat16,fp8,0,0.1343456029891968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,1,128,1,bfloat16,bfloat16,0,0.023766399919986726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,1,128,1,bfloat16,fp8,0,0.025128000974655153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,1,128,1,fp8,fp8,0,0.02513439953327179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,2,128,1,bfloat16,bfloat16,0,0.02364639937877655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,2,128,1,bfloat16,fp8,0,0.025260800123214723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,2,128,1,fp8,fp8,0,0.02499839961528778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,4,128,1,bfloat16,bfloat16,0,0.02698880136013031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,4,128,1,bfloat16,fp8,0,0.0291920006275177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,4,128,1,fp8,fp8,0,0.029335999488830568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,8,128,1,bfloat16,bfloat16,0,0.028137600421905516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,8,128,1,bfloat16,fp8,0,0.02953599989414215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,8,128,1,fp8,fp8,0,0.02927359938621521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,48,128,1,bfloat16,bfloat16,0,0.06320319771766662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,48,128,1,bfloat16,fp8,0,0.06513280272483826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,48,128,1,fp8,fp8,0,0.0658847987651825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,1,128,1,bfloat16,bfloat16,0,0.017748799920082093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,48,4,128,1,bfloat16,bfloat16,0,0.01252640038728714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,1,128,1,bfloat16,fp8,0,0.01889120042324066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,1,128,1,fp8,fp8,0,0.019017599523067474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,2,128,1,bfloat16,bfloat16,0,0.018267199397087097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,2,128,1,bfloat16,fp8,0,0.018907199800014495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,2,128,1,fp8,fp8,0,0.018982400000095368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,4,128,1,bfloat16,bfloat16,0,0.018083199858665466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,4,128,1,bfloat16,fp8,0,0.018881599605083465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,4,128,1,fp8,fp8,0,0.019120000302791595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,8,128,1,bfloat16,bfloat16,0,0.018423999845981597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,8,128,1,bfloat16,fp8,0,0.01891999989748001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,48,8,128,1,fp8,fp8,0,0.018913599848747253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,48,128,1,bfloat16,bfloat16,0,0.033502399921417236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,48,128,1,bfloat16,fp8,0,0.03693279922008515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,48,128,1,fp8,fp8,0,0.03670719861984253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,1,128,1,bfloat16,fp8,0,0.01371999979019165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,48,8,128,1,fp8,fp8,0,0.012988799810409546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,1,128,1,fp8,fp8,0,0.013683199882507324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,2,128,1,bfloat16,bfloat16,0,0.0131632000207901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,2,128,1,fp8,fp8,0,0.013654400408267976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,4,128,1,bfloat16,bfloat16,0,0.013081599771976472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,4,128,1,bfloat16,fp8,0,0.01363680064678192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,4,128,1,fp8,fp8,0,0.01361439973115921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,8,128,1,bfloat16,bfloat16,0,0.013251200318336487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,8,128,1,bfloat16,fp8,0,0.013683199882507324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,8,128,1,fp8,fp8,0,0.013857600092887879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,48,128,1,bfloat16,bfloat16,0,0.021169599890708924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,48,128,1,bfloat16,fp8,0,0.02253119945526123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,48,128,1,fp8,fp8,0,0.022563199698925018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,1,128,1,bfloat16,bfloat16,0,0.012691199779510498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,1,128,1,bfloat16,fp8,0,0.013291199505329133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,1,128,1,fp8,fp8,0,0.013300800323486328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,2,128,1,bfloat16,bfloat16,0,0.01268640011548996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,2,128,1,bfloat16,fp8,0,0.013369600474834441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,8,128,1,fp8,fp8,0,0.05918239951133728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,2,128,1,fp8,fp8,0,0.0133200004696846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,4,128,1,bfloat16,bfloat16,0,0.012788799405097962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,4,128,1,bfloat16,fp8,0,0.013283200562000275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,48,48,128,1,fp8,fp8,0,0.13340959548950196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,8,128,1,bfloat16,bfloat16,0,0.012454400211572647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,8,128,1,fp8,fp8,0,0.013124799728393555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,48,128,1,bfloat16,bfloat16,0,0.016843199729919434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,48,128,1,bfloat16,fp8,0,0.017609600722789765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,48,128,1,fp8,fp8,0,0.017679999768733978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,1,128,1,bfloat16,bfloat16,0,0.01231039986014366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,1,128,1,bfloat16,fp8,0,0.013016000390052795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,1,128,1,fp8,fp8,0,0.013102400302886962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,2,128,1,bfloat16,bfloat16,0,0.012611199915409089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,2,128,1,bfloat16,fp8,0,0.013102400302886962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,2,128,1,fp8,fp8,0,0.01313759982585907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,4,128,1,bfloat16,bfloat16,0,0.012371200323104858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,4,128,1,bfloat16,fp8,0,0.01308639943599701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,4,128,1,fp8,fp8,0,0.013196800649166108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,8,128,1,bfloat16,bfloat16,0,0.012355200201272964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,8,128,1,bfloat16,fp8,0,0.012982399761676788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,48,8,128,1,fp8,fp8,0,0.012913599610328674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,48,128,1,bfloat16,bfloat16,0,0.013059200346469879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,48,128,1,bfloat16,fp8,0,0.013257600367069244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,48,128,1,fp8,fp8,0,0.013289600610733032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,1,128,1,bfloat16,bfloat16,0,0.012108799815177918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,1,128,1,bfloat16,fp8,0,0.012700800597667695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,1,128,1,fp8,fp8,0,0.01276479959487915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,2,128,1,bfloat16,bfloat16,0,0.012145599722862244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,2,128,1,bfloat16,fp8,0,0.012652799487113953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,2,128,1,fp8,fp8,0,0.012859199941158295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,4,128,1,bfloat16,bfloat16,0,0.012176000326871873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,1,128,1,bfloat16,bfloat16,0,0.012891200184822083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,4,128,1,fp8,fp8,0,0.012913599610328674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,8,128,1,bfloat16,bfloat16,0,0.012220799922943115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,8,128,1,bfloat16,fp8,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,48,2,128,1,bfloat16,fp8,0,0.013734400272369385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,48,128,1,bfloat16,bfloat16,0,0.013171200454235078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,48,128,1,fp8,fp8,0,0.013393600285053254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,1,128,1,bfloat16,bfloat16,0,0.011990399658679962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,1,128,1,bfloat16,fp8,0,0.01260959953069687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,1,128,1,fp8,fp8,0,0.012521600723266602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,2,128,1,bfloat16,bfloat16,0,0.01194560006260872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,2,128,1,bfloat16,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,2,128,1,fp8,fp8,0,0.012591999769210816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,4,128,1,bfloat16,bfloat16,0,0.011932799965143204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,4,128,1,bfloat16,fp8,0,0.012555199861526489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,4,128,1,fp8,fp8,0,0.012614400684833526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,8,128,1,bfloat16,bfloat16,0,0.011948800086975098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,8,128,1,bfloat16,fp8,0,0.012639999389648438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,8,128,1,fp8,fp8,0,0.012624000012874604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,1,128,1,bfloat16,bfloat16,0,0.021428799629211424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,48,8,128,1,bfloat16,bfloat16,0,0.06020960211753845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,1,128,1,fp8,fp8,0,0.023108799755573273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,4,128,1,fp8,fp8,0,0.013460800051689148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,2,128,1,bfloat16,bfloat16,0,0.028295999765396117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,2,128,1,bfloat16,fp8,0,0.03118560016155243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,2,128,1,fp8,fp8,0,0.03162400126457214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,4,128,1,bfloat16,bfloat16,0,0.04277600049972534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,4,128,1,bfloat16,fp8,0,0.049246400594711304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,4,128,1,fp8,fp8,0,0.048844799399375916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,8,128,1,bfloat16,bfloat16,0,0.07092959880828857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,8,128,1,bfloat16,fp8,0,0.08374879956245422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,8,128,1,fp8,fp8,0,0.08422240018844604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,48,128,1,bfloat16,bfloat16,0,0.1822111964225769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,48,128,1,bfloat16,fp8,0,0.2236255884170532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,1,128,1,bfloat16,bfloat16,0,0.014569599926471711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,48,128,1,fp8,fp8,0,0.22279839515686034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,1,128,1,bfloat16,fp8,0,0.01581439971923828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,1,128,1,fp8,fp8,0,0.016014400124549865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,2,128,1,bfloat16,bfloat16,0,0.018328000605106354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,2,128,1,fp8,fp8,0,0.020428800582885744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,2,128,1,bfloat16,fp8,0,0.020113599300384522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,4,128,1,bfloat16,bfloat16,0,0.026044800877571106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,4,128,1,bfloat16,fp8,0,0.029207998514175416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,4,128,1,fp8,fp8,0,0.029067200422286988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,8,128,1,bfloat16,bfloat16,0,0.04048640131950378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,8,128,1,bfloat16,fp8,0,0.04671039879322052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,48,8,128,1,fp8,fp8,0,0.046563199162483214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,48,128,1,bfloat16,bfloat16,0,0.09834240078926086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,8,128,1,fp8,fp8,0,0.012691199779510498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,1,128,1,bfloat16,bfloat16,0,0.013246400654315949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,1,128,1,bfloat16,fp8,0,0.014428800344467163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,48,48,128,1,bfloat16,fp8,0,0.013340799510478974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,1,128,1,fp8,fp8,0,0.014086399972438813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,2,128,1,bfloat16,bfloat16,0,0.013417600095272065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,2,128,1,bfloat16,fp8,0,0.014212800562381745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,4,128,1,bfloat16,bfloat16,0,0.01740480065345764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,4,128,1,bfloat16,fp8,0,0.018913599848747253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,4,128,1,fp8,fp8,0,0.01897599995136261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,8,128,1,bfloat16,bfloat16,0,0.024879999458789825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,8,128,1,bfloat16,fp8,0,0.027667200565338133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,8,128,1,fp8,fp8,0,0.02770560085773468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,48,128,1,bfloat16,bfloat16,0,0.054763197898864746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,48,128,1,bfloat16,fp8,0,0.06382880210876465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,48,128,1,fp8,fp8,0,0.06383519768714904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,1,128,1,bfloat16,bfloat16,0,0.012771199643611907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,1,128,1,bfloat16,fp8,0,0.013612799346446991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,48,1,128,1,bfloat16,fp8,0,0.023001599311828613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,1,128,1,fp8,fp8,0,0.013366399705410004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,2,128,1,bfloat16,bfloat16,0,0.012761600315570831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,2,128,1,bfloat16,fp8,0,0.013566400110721587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,2,128,1,fp8,fp8,0,0.013447999954223633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,4,128,1,bfloat16,bfloat16,0,0.01268800050020218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,4,128,1,bfloat16,fp8,0,0.013510400056838989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,48,8,128,1,bfloat16,fp8,0,0.013097600638866424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,4,128,1,fp8,fp8,0,0.013558399677276612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,8,128,1,bfloat16,bfloat16,0,0.016923199594020843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,8,128,1,fp8,fp8,0,0.017875200510025023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,48,8,128,1,bfloat16,fp8,0,0.018063999712467194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,48,128,1,bfloat16,bfloat16,0,0.0324288010597229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,48,128,1,bfloat16,fp8,0,0.035899201035499574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,1,128,1,bfloat16,bfloat16,0,0.012083200365304947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,1,128,1,bfloat16,fp8,0,0.012857599556446076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,1,128,1,fp8,fp8,0,0.01276479959487915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,2,128,1,bfloat16,bfloat16,0,0.012163200229406358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,2,128,1,bfloat16,fp8,0,0.012945599853992462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,2,128,1,fp8,fp8,0,0.013084800541400909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,4,128,1,bfloat16,fp8,0,0.013120000064373017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,4,128,1,fp8,fp8,0,0.013081599771976472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,8,128,1,bfloat16,bfloat16,0,0.012212800234556198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,8,128,1,bfloat16,fp8,0,0.013072000443935394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,8,128,1,fp8,fp8,0,0.013096000254154205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,48,128,1,bfloat16,bfloat16,0,0.021028800308704375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,48,128,1,bfloat16,fp8,0,0.022380800545215608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,48,128,1,fp8,fp8,0,0.022515200078487396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,48,4,128,1,bfloat16,fp8,0,0.013019199669361114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,1,128,1,bfloat16,bfloat16,0,0.012118399888277055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,1,128,1,bfloat16,fp8,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,2,128,1,bfloat16,bfloat16,0,0.01215519979596138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,48,128,1,bfloat16,fp8,0,0.11786240339279175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,2,128,1,bfloat16,fp8,0,0.012891200184822083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,48,128,1,fp8,fp8,0,0.118339204788208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,4,128,1,bfloat16,bfloat16,0,0.0122079998254776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,4,128,1,bfloat16,fp8,0,0.012726399302482604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,48,2,128,1,fp8,fp8,0,0.014163200557231904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,4,128,1,fp8,fp8,0,0.01290079951286316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,8,128,1,bfloat16,bfloat16,0,0.012038400024175644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,8,128,1,bfloat16,fp8,0,0.012822400033473968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,8,128,1,fp8,fp8,0,0.012868799269199371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,48,128,1,bfloat16,bfloat16,0,0.01666239947080612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,48,128,1,bfloat16,fp8,0,0.01733119934797287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,48,128,1,fp8,fp8,0,0.017529599368572235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,1,128,1,bfloat16,bfloat16,0,0.01162400022149086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,1,128,1,bfloat16,fp8,0,0.012220799922943115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,1,128,1,fp8,fp8,0,0.01239520013332367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,2,128,1,bfloat16,bfloat16,0,0.011671999841928482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,2,128,1,bfloat16,fp8,0,0.012179200351238251
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,2,128,1,fp8,fp8,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,4,128,1,bfloat16,fp8,0,0.012427199631929398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,4,128,1,fp8,fp8,0,0.012638400495052337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,8,128,1,bfloat16,bfloat16,0,0.01188960000872612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,8,128,1,bfloat16,fp8,0,0.012873600423336028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,8,128,1,fp8,fp8,0,0.012505599856376648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,48,128,1,bfloat16,bfloat16,0,0.013084800541400909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,48,128,1,bfloat16,fp8,0,0.01327199935913086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,48,128,1,fp8,fp8,0,0.01342879980802536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,1,128,1,bfloat16,bfloat16,0,0.011611200124025344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,1,128,1,bfloat16,fp8,0,0.0124208003282547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,1,128,1,fp8,fp8,0,0.012263999879360199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,2,128,1,bfloat16,bfloat16,0,0.011760000139474869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,2,128,1,bfloat16,fp8,0,0.01223680004477501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,2,128,1,fp8,fp8,0,0.012411200255155564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,4,128,1,bfloat16,bfloat16,0,0.011488000303506852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,4,128,1,bfloat16,fp8,0,0.012455999851226807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,4,128,1,fp8,fp8,0,0.012240000069141388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,8,128,1,bfloat16,bfloat16,0,0.011751999706029892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,8,128,1,bfloat16,fp8,0,0.012238399684429168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,48,8,128,1,fp8,fp8,0,0.012427199631929398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,48,128,1,bfloat16,bfloat16,0,0.012647999823093415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,48,128,1,bfloat16,fp8,0,0.01308320015668869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,48,128,1,fp8,fp8,0,0.012895999848842621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,1,128,1,bfloat16,bfloat16,0,0.011684799939393998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,1,128,1,bfloat16,fp8,0,0.012104000151157378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,1,128,1,fp8,fp8,0,0.012430399656295776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,2,128,1,bfloat16,bfloat16,0,0.011385600268840789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,2,128,1,bfloat16,fp8,0,0.0124208003282547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,48,128,1,fp8,fp8,0,0.03611040115356445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,2,128,1,fp8,fp8,0,0.012572799623012543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,4,128,1,bfloat16,bfloat16,0,0.011604800075292587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,4,128,1,bfloat16,fp8,0,0.012263999879360199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,4,128,1,fp8,fp8,0,0.012302400171756744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,8,128,1,bfloat16,bfloat16,0,0.011718399822711945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,8,128,1,bfloat16,fp8,0,0.012451200187206269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,48,8,128,1,fp8,fp8,0,0.012318400293588638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,48,4,128,1,bfloat16,bfloat16,0,0.012267199903726577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,1,128,1,fp8,fp8,0,0.012878400087356568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,48,2,128,1,fp8,fp8,0,0.012591999769210816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,48,4,128,1,bfloat16,bfloat16,0,0.011662399768829346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,1,128,1,fp8,fp8,0,31.50609130859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,1,128,1,bfloat16,fp8,0,31.584939575195314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,2,128,1,bfloat16,fp8,0,30.99663391113281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,2,128,1,fp8,fp8,0,30.994900512695313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,2,128,1,bfloat16,bfloat16,0,37.46196594238281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,1,128,1,bfloat16,bfloat16,0,38.06807861328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,4,128,1,bfloat16,fp8,0,31.31157531738281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,4,128,1,bfloat16,bfloat16,0,38.14065856933594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,1,128,1,bfloat16,bfloat16,0,18.37712860107422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,4,128,1,fp8,fp8,0,33.05458984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,8,128,1,fp8,fp8,0,31.691644287109376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,8,128,1,bfloat16,fp8,0,32.368386840820314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,40,8,128,1,bfloat16,bfloat16,0,38.54491271972656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,1,128,1,bfloat16,fp8,0,15.981852722167968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,1,128,1,fp8,fp8,0,16.49993896484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,2,128,1,bfloat16,fp8,0,16.969517517089844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,2,128,1,bfloat16,bfloat16,0,20.331011962890624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,2,128,1,fp8,fp8,0,16.765867614746092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,4,128,1,bfloat16,bfloat16,0,19.142910766601563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,4,128,1,fp8,fp8,0,15.936471557617187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,4,128,1,bfloat16,fp8,0,16.781352233886718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,8,128,1,bfloat16,bfloat16,0,18.45142059326172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,8,128,1,bfloat16,fp8,0,16.112318420410155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,8,128,1,fp8,fp8,0,15.760212707519532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,1,128,1,bfloat16,bfloat16,0,10.383782196044923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,1,128,1,bfloat16,fp8,0,8.231919860839843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,1,128,1,fp8,fp8,0,8.118344116210938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,2,128,1,bfloat16,bfloat16,0,9.609449768066407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,2,128,1,bfloat16,fp8,0,7.878009796142578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,2,128,1,fp8,fp8,0,7.891500854492188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,4,128,1,bfloat16,bfloat16,0,9.706501007080078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,4,128,1,bfloat16,fp8,0,7.82336196899414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,4,128,1,fp8,fp8,0,8.029190063476562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,8,128,1,bfloat16,bfloat16,0,10.130404663085937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,8,128,1,bfloat16,fp8,0,8.66901092529297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,8,128,1,fp8,fp8,0,7.9634254455566404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,40,128,1,bfloat16,fp8,0,4.022024154663086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,40,128,1,bfloat16,bfloat16,0,4.8754016876220705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,40,128,1,bfloat16,fp8,0,8.049327850341797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,40,128,1,fp8,fp8,0,8.039141082763672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,40,128,1,fp8,fp8,0,4.026655960083008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,1,128,1,bfloat16,fp8,0,3.8381584167480467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,1,128,1,bfloat16,bfloat16,0,4.666864013671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,40,40,128,1,bfloat16,bfloat16,0,9.516659545898438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,1,128,1,fp8,fp8,0,3.835785675048828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,40,128,1,bfloat16,fp8,0,16.0047119140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,2,128,1,bfloat16,bfloat16,0,4.521067047119141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,2,128,1,bfloat16,fp8,0,3.8567886352539062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,2,128,1,fp8,fp8,0,4.030176162719727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,4,128,1,bfloat16,fp8,0,3.873972702026367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,4,128,1,bfloat16,bfloat16,0,4.484137725830078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,40,128,1,fp8,fp8,0,15.97821502685547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,40,40,128,1,bfloat16,bfloat16,0,19.040864562988283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,4,128,1,fp8,fp8,0,3.88316650390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,8,128,1,bfloat16,fp8,0,3.9006847381591796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,8,128,1,bfloat16,bfloat16,0,4.619315338134766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,40,8,128,1,fp8,fp8,0,3.9415550231933594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,1,128,1,bfloat16,fp8,0,17.643295288085938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,1,128,1,fp8,fp8,0,18.296279907226562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,2,128,1,bfloat16,fp8,0,19.181642150878908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,1,128,1,bfloat16,bfloat16,0,21.82744445800781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,2,128,1,fp8,fp8,0,19.34545135498047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,4,128,1,bfloat16,fp8,0,18.909587097167968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,2,128,1,bfloat16,bfloat16,0,22.578094482421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,4,128,1,bfloat16,bfloat16,0,22.890872192382812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,40,128,1,bfloat16,fp8,0,9.907987213134765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,40,128,1,fp8,fp8,0,9.154370880126953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,40,128,1,bfloat16,bfloat16,0,11.578385925292968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,1,128,1,bfloat16,bfloat16,0,10.410791778564453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,4,128,1,fp8,fp8,0,19.311308288574217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,8,128,1,bfloat16,fp8,0,18.004835510253905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,8,128,1,fp8,fp8,0,17.81225128173828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,40,8,128,1,bfloat16,bfloat16,0,21.785977172851563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,1,128,1,bfloat16,fp8,0,9.072051239013671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,2,128,1,bfloat16,bfloat16,0,10.47952651977539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,2,128,1,bfloat16,fp8,0,8.816417694091797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,1,128,1,fp8,fp8,0,8.750566101074218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,2,128,1,fp8,fp8,0,8.999667358398437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,4,128,1,bfloat16,fp8,0,8.835750579833984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,4,128,1,bfloat16,bfloat16,0,10.940030670166015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,4,128,1,fp8,fp8,0,8.864435577392578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,40,128,1,bfloat16,bfloat16,0,5.490678405761718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,40,128,1,bfloat16,fp8,0,4.638166427612305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,8,128,1,bfloat16,bfloat16,0,11.154241943359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,8,128,1,bfloat16,fp8,0,9.479153442382813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,40,128,1,fp8,fp8,0,4.887547302246094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,40,8,128,1,fp8,fp8,0,8.989713287353515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,1,128,1,bfloat16,fp8,0,4.575088119506836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,1,128,1,bfloat16,bfloat16,0,5.623006439208984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,1,128,1,fp8,fp8,0,4.521209716796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,2,128,1,bfloat16,bfloat16,0,5.2603199005126955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,2,128,1,bfloat16,fp8,0,4.609113693237305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,2,128,1,fp8,fp8,0,4.807849502563476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,4,128,1,bfloat16,fp8,0,4.822003173828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,4,128,1,bfloat16,bfloat16,0,6.066867065429688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,4,128,1,fp8,fp8,0,4.833486557006836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,8,128,1,bfloat16,fp8,0,4.4316143035888675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,40,128,1,bfloat16,bfloat16,0,2.989409637451172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,40,128,1,bfloat16,fp8,0,2.4693119049072267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,8,128,1,bfloat16,bfloat16,0,5.799486541748047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,40,8,128,1,fp8,fp8,0,4.674441528320313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,40,128,1,fp8,fp8,0,2.5972383499145506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,1,128,1,fp8,fp8,0,2.1935104370117187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,1,128,1,bfloat16,fp8,0,2.3560848236083984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,1,128,1,bfloat16,bfloat16,0,2.932516860961914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,2,128,1,bfloat16,bfloat16,0,2.6925039291381836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,2,128,1,bfloat16,fp8,0,2.483367919921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,2,128,1,fp8,fp8,0,2.358576011657715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,4,128,1,bfloat16,bfloat16,0,2.741017532348633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,4,128,1,bfloat16,fp8,0,2.2146656036376955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,4,128,1,fp8,fp8,0,2.3211343765258787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,8,128,1,bfloat16,bfloat16,0,2.659377670288086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,8,128,1,fp8,fp8,0,2.226631927490234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,40,8,128,1,bfloat16,fp8,0,2.484139251708984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,1,128,1,fp8,fp8,0,12.339694213867187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,1,128,1,bfloat16,fp8,0,12.550550079345703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,2,128,1,bfloat16,fp8,0,12.38007049560547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,2,128,1,fp8,fp8,0,12.33890380859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,1,128,1,bfloat16,bfloat16,0,15.406745910644531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,2,128,1,bfloat16,bfloat16,0,15.202542114257813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,4,128,1,bfloat16,bfloat16,0,15.667303466796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,4,128,1,bfloat16,fp8,0,12.669305419921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,40,128,1,bfloat16,fp8,0,6.596656036376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,40,128,1,bfloat16,bfloat16,0,7.978849792480469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,40,128,1,fp8,fp8,0,6.763132476806641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,1,128,1,bfloat16,bfloat16,0,7.475718688964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,4,128,1,fp8,fp8,0,13.349940490722656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,8,128,1,bfloat16,fp8,0,12.70806427001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,8,128,1,fp8,fp8,0,13.130349731445312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,40,8,128,1,bfloat16,bfloat16,0,15.37891082763672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,1,128,1,bfloat16,fp8,0,6.302896118164062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,1,128,1,fp8,fp8,0,6.2570640563964846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,2,128,1,bfloat16,fp8,0,6.317528152465821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,2,128,1,bfloat16,bfloat16,0,7.857654571533203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,2,128,1,fp8,fp8,0,6.209531021118164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,4,128,1,bfloat16,fp8,0,6.745022583007812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,4,128,1,fp8,fp8,0,6.525163269042968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,40,128,1,bfloat16,fp8,0,3.2648433685302733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,8,128,1,bfloat16,fp8,0,6.532185363769531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,40,128,1,bfloat16,bfloat16,0,4.116123199462891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,8,128,1,bfloat16,bfloat16,0,8.419921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,40,128,1,fp8,fp8,0,3.466056060791016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,8,128,1,fp8,fp8,0,6.5569091796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,40,4,128,1,bfloat16,bfloat16,0,8.525516510009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,1,128,1,bfloat16,bfloat16,0,3.7703697204589846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,1,128,1,bfloat16,fp8,0,3.080531120300293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,1,128,1,fp8,fp8,0,3.232467269897461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,2,128,1,bfloat16,fp8,0,3.213230514526367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,2,128,1,bfloat16,bfloat16,0,3.7729297637939454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,2,128,1,fp8,fp8,0,3.115737533569336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,4,128,1,bfloat16,fp8,0,3.11661434173584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,4,128,1,fp8,fp8,0,3.5018192291259767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,4,128,1,bfloat16,bfloat16,0,4.219014358520508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,40,128,1,bfloat16,bfloat16,0,2.100691223144531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,40,128,1,bfloat16,fp8,0,1.6597007751464843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,8,128,1,bfloat16,fp8,0,3.3035919189453127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,8,128,1,bfloat16,bfloat16,0,3.7802337646484374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,40,128,1,fp8,fp8,0,1.677801513671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,40,8,128,1,fp8,fp8,0,3.1201055526733397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,1,128,1,bfloat16,bfloat16,0,1.7308784484863282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,1,128,1,bfloat16,fp8,0,2.03875675201416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,2,128,1,bfloat16,fp8,0,1.6142671585083008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,2,128,1,bfloat16,bfloat16,0,1.7143840789794922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,1,128,1,fp8,fp8,0,1.7137008666992188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,2,128,1,fp8,fp8,0,1.917643165588379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,4,128,1,bfloat16,fp8,0,1.5577759742736816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,4,128,1,bfloat16,bfloat16,0,1.6923471450805665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,4,128,1,fp8,fp8,0,1.7242416381835937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,8,128,1,bfloat16,fp8,0,1.6299135208129882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,8,128,1,fp8,fp8,0,1.7662752151489258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,40,8,128,1,bfloat16,bfloat16,0,2.0018287658691407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,1,128,1,bfloat16,fp8,0,16.624919128417968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,1,128,1,fp8,fp8,0,16.650038146972655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,2,128,1,bfloat16,fp8,0,16.215290832519532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,2,128,1,fp8,fp8,0,16.691473388671874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,4,128,1,bfloat16,fp8,0,16.55912628173828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,2,128,1,bfloat16,bfloat16,0,18.852499389648436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,1,128,1,bfloat16,bfloat16,0,19.847122192382812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,4,128,1,bfloat16,bfloat16,0,19.715252685546876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,40,128,1,bfloat16,fp8,0,9.29787826538086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,40,128,1,fp8,fp8,0,8.923560333251952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,40,128,1,bfloat16,bfloat16,0,10.958493041992188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,1,128,1,bfloat16,bfloat16,0,10.164828491210937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,4,128,1,fp8,fp8,0,16.285275268554688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,8,128,1,bfloat16,fp8,0,17.30020751953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,8,128,1,fp8,fp8,0,17.145872497558592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,40,8,128,1,bfloat16,bfloat16,0,20.433499145507813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,1,128,1,bfloat16,fp8,0,8.304366302490234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,1,128,1,fp8,fp8,0,8.448395538330079
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,2,128,1,bfloat16,fp8,0,8.219556427001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,2,128,1,bfloat16,bfloat16,0,10.619048309326171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,2,128,1,fp8,fp8,0,8.289329528808594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,4,128,1,bfloat16,fp8,0,8.450593566894531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,4,128,1,bfloat16,bfloat16,0,9.98852767944336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,40,128,1,bfloat16,bfloat16,0,5.39990234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,4,128,1,fp8,fp8,0,8.158905792236329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,40,128,1,bfloat16,fp8,0,4.3734081268310545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,8,128,1,bfloat16,fp8,0,8.723648071289062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,8,128,1,bfloat16,bfloat16,0,9.774320220947265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,40,8,128,1,fp8,fp8,0,8.15352783203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,40,128,1,fp8,fp8,0,4.392316818237305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,1,128,1,bfloat16,bfloat16,0,4.695832061767578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,1,128,1,fp8,fp8,0,4.043633651733399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,1,128,1,bfloat16,fp8,0,4.101206588745117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,2,128,1,bfloat16,bfloat16,0,4.982372665405274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,2,128,1,bfloat16,fp8,0,4.111619186401367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,2,128,1,fp8,fp8,0,4.080412673950195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,4,128,1,bfloat16,bfloat16,0,5.123459243774414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,4,128,1,bfloat16,fp8,0,4.1088207244873045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,4,128,1,fp8,fp8,0,4.126639938354492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,8,128,1,bfloat16,bfloat16,0,5.0119281768798825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,40,128,1,bfloat16,bfloat16,0,2.6516128540039063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,8,128,1,bfloat16,fp8,0,4.311505508422852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,40,8,128,1,fp8,fp8,0,4.0752910614013675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,40,128,1,bfloat16,fp8,0,2.418339157104492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,40,128,1,fp8,fp8,0,2.1495840072631838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,1,128,1,bfloat16,fp8,0,2.0363935470581054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,1,128,1,bfloat16,bfloat16,0,2.271774482727051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,1,128,1,fp8,fp8,0,2.2315439224243163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,2,128,1,bfloat16,fp8,0,2.020809555053711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,2,128,1,bfloat16,bfloat16,0,2.5680688858032226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,2,128,1,fp8,fp8,0,2.0233743667602537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,4,128,1,bfloat16,bfloat16,0,2.218361663818359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,4,128,1,fp8,fp8,0,2.0600048065185548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,8,128,1,bfloat16,bfloat16,0,2.4288272857666016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,40,128,1,bfloat16,fp8,0,1.0820367813110352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,40,128,1,bfloat16,bfloat16,0,1.3422559738159179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,8,128,1,bfloat16,fp8,0,2.040492820739746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,40,128,1,fp8,fp8,0,1.156007957458496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,8,128,1,fp8,fp8,0,2.0402591705322264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,1,128,1,bfloat16,bfloat16,0,1.1293871879577637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,1,128,1,fp8,fp8,0,1.009432029724121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,1,128,1,bfloat16,fp8,0,1.1554816246032715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,2,128,1,bfloat16,bfloat16,0,1.278264045715332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,2,128,1,bfloat16,fp8,0,1.0498687744140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,2,128,1,fp8,fp8,0,1.0864319801330566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,4,128,1,bfloat16,bfloat16,0,1.097372817993164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,4,128,1,bfloat16,fp8,0,1.1521103858947754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,4,128,1,fp8,fp8,0,1.01942720413208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,8,128,1,bfloat16,bfloat16,0,1.1318863868713378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,8,128,1,bfloat16,fp8,0,1.1936911582946776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,40,8,128,1,fp8,fp8,0,1.0637935638427733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,40,4,128,1,bfloat16,fp8,0,2.039041519165039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,1,128,1,bfloat16,fp8,0,9.362671661376954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,1,128,1,fp8,fp8,0,9.735441589355469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,2,128,1,bfloat16,fp8,0,9.639974212646484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,2,128,1,fp8,fp8,0,9.7186767578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,4,128,1,bfloat16,fp8,0,9.379507446289063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,1,128,1,bfloat16,bfloat16,0,11.322417449951171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,2,128,1,bfloat16,bfloat16,0,11.939151763916016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,4,128,1,bfloat16,bfloat16,0,11.55810546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,40,128,1,bfloat16,fp8,0,5.187862396240234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,40,128,1,bfloat16,bfloat16,0,6.196289443969727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,40,128,1,fp8,fp8,0,5.063710403442383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,1,128,1,bfloat16,bfloat16,0,5.957491302490235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,4,128,1,fp8,fp8,0,9.43704833984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,8,128,1,fp8,fp8,0,9.56468505859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,8,128,1,bfloat16,fp8,0,10.00540771484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,1,128,1,bfloat16,fp8,0,4.707825469970703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,40,8,128,1,bfloat16,bfloat16,0,11.936830139160156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,1,128,1,fp8,fp8,0,4.738518524169922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,2,128,1,bfloat16,bfloat16,0,5.852830505371093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,2,128,1,bfloat16,fp8,0,4.961601638793946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,2,128,1,fp8,fp8,0,4.686342239379883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,4,128,1,bfloat16,fp8,0,5.1159008026123045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,4,128,1,bfloat16,bfloat16,0,5.602030563354492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,4,128,1,fp8,fp8,0,4.806584167480469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,40,128,1,bfloat16,fp8,0,2.6311599731445314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,40,128,1,bfloat16,bfloat16,0,3.157244873046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,8,128,1,bfloat16,fp8,0,4.7261711120605465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,8,128,1,bfloat16,bfloat16,0,5.649849700927734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,40,8,128,1,fp8,fp8,0,4.764188766479492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,40,128,1,fp8,fp8,0,2.7470575332641602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,1,128,1,bfloat16,bfloat16,0,2.932217597961426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,1,128,1,bfloat16,fp8,0,2.4571632385253905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,1,128,1,fp8,fp8,0,2.446945571899414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,2,128,1,bfloat16,fp8,0,2.339627265930176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,2,128,1,bfloat16,bfloat16,0,2.8520944595336912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,2,128,1,fp8,fp8,0,2.4423711776733397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,4,128,1,bfloat16,bfloat16,0,2.6912736892700195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,4,128,1,bfloat16,fp8,0,2.422209548950195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,40,128,1,bfloat16,bfloat16,0,1.5839103698730468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,40,128,1,bfloat16,fp8,0,1.2801584243774413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,4,128,1,fp8,fp8,0,2.6031904220581055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,8,128,1,bfloat16,bfloat16,0,2.8187408447265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,8,128,1,bfloat16,fp8,0,2.432614326477051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,40,8,128,1,fp8,fp8,0,2.379167938232422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,40,128,1,fp8,fp8,0,1.4602144241333008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,1,128,1,bfloat16,bfloat16,0,1.5828495979309083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,1,128,1,fp8,fp8,0,1.1694095611572266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,1,128,1,bfloat16,fp8,0,1.2998288154602051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,2,128,1,bfloat16,bfloat16,0,1.2888575553894044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,2,128,1,fp8,fp8,0,1.1742639541625977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,2,128,1,bfloat16,fp8,0,1.3989472389221191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,4,128,1,bfloat16,bfloat16,0,1.344324779510498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,40,128,1,bfloat16,bfloat16,0,0.7370704174041748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,4,128,1,bfloat16,fp8,0,1.250870418548584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,40,128,1,bfloat16,fp8,0,0.733128023147583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,8,128,1,bfloat16,bfloat16,0,1.2860223770141601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,8,128,1,bfloat16,fp8,0,1.2225088119506835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,8,128,1,fp8,fp8,0,1.3424176216125487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,40,128,1,fp8,fp8,0,0.7939328193664551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,1,128,1,bfloat16,bfloat16,0,0.6283152103424072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,1,128,1,bfloat16,fp8,0,0.6136559963226318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,1,128,1,fp8,fp8,0,0.5899360179901123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,2,128,1,bfloat16,bfloat16,0,0.6480112075805664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,2,128,1,bfloat16,fp8,0,0.6528416156768799
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,2,128,1,fp8,fp8,0,0.7027584075927734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,4,128,1,bfloat16,bfloat16,0,0.6339536190032959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,4,128,1,bfloat16,fp8,0,0.6132912158966064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,4,128,1,fp8,fp8,0,0.6069583892822266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,8,128,1,bfloat16,bfloat16,0,0.6612559795379639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,8,128,1,bfloat16,fp8,0,0.6399312019348145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,40,8,128,1,fp8,fp8,0,0.6735151767730713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,40,4,128,1,fp8,fp8,0,1.3486847877502441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,1,128,1,bfloat16,fp8,0,8.752120208740234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,1,128,1,fp8,fp8,0,8.93930435180664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,2,128,1,bfloat16,fp8,0,8.957472229003907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,2,128,1,fp8,fp8,0,8.9414306640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,1,128,1,bfloat16,bfloat16,0,10.756719970703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,2,128,1,bfloat16,bfloat16,0,10.512633514404296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,4,128,1,bfloat16,bfloat16,0,10.524713897705078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,4,128,1,bfloat16,fp8,0,8.858939361572265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,40,128,1,bfloat16,fp8,0,5.084836959838867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,40,128,1,bfloat16,bfloat16,0,5.846247863769531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,4,128,1,fp8,fp8,0,9.018891143798829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,40,128,1,fp8,fp8,0,4.909983825683594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,8,128,1,bfloat16,fp8,0,9.049330902099609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,8,128,1,fp8,fp8,0,9.103787231445313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,40,8,128,1,bfloat16,bfloat16,0,11.3071533203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,1,128,1,bfloat16,bfloat16,0,5.1460014343261715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,1,128,1,bfloat16,fp8,0,4.6273551940917965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,1,128,1,fp8,fp8,0,4.573921585083008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,2,128,1,bfloat16,fp8,0,4.775408172607422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,2,128,1,fp8,fp8,0,4.383876800537109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,2,128,1,bfloat16,bfloat16,0,5.484659194946289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,4,128,1,bfloat16,bfloat16,0,5.219363021850586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,4,128,1,bfloat16,fp8,0,4.452737426757812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,4,128,1,fp8,fp8,0,4.490224075317383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,8,128,1,bfloat16,fp8,0,4.465335845947266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,40,128,1,bfloat16,fp8,0,2.654220771789551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,40,128,1,bfloat16,bfloat16,0,3.0106239318847656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,8,128,1,bfloat16,bfloat16,0,5.494225692749024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,40,128,1,fp8,fp8,0,2.6978368759155273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,1,128,1,bfloat16,bfloat16,0,2.6281551361083983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,1,128,1,bfloat16,fp8,0,2.1943391799926757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,40,8,128,1,fp8,fp8,0,4.886995315551758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,1,128,1,fp8,fp8,0,2.192379188537598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,2,128,1,bfloat16,bfloat16,0,2.356228828430176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,2,128,1,bfloat16,fp8,0,2.4321407318115233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,4,128,1,bfloat16,bfloat16,0,2.5220703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,4,128,1,fp8,fp8,0,2.2178592681884766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,4,128,1,bfloat16,fp8,0,2.5035024642944337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,40,128,1,bfloat16,bfloat16,0,1.4165871620178223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,8,128,1,bfloat16,bfloat16,0,2.660176086425781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,8,128,1,bfloat16,fp8,0,2.250588798522949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,8,128,1,fp8,fp8,0,2.4608575820922853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,40,128,1,bfloat16,fp8,0,1.3907759666442872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,40,128,1,fp8,fp8,0,1.236025619506836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,1,128,1,bfloat16,bfloat16,0,1.1648480415344238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,1,128,1,bfloat16,fp8,0,1.158795166015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,2,128,1,bfloat16,bfloat16,0,1.2081279754638672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,1,128,1,fp8,fp8,0,1.2631888389587402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,2,128,1,fp8,fp8,0,1.1316240310668946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,2,128,1,bfloat16,fp8,0,1.137281608581543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,40,2,128,1,fp8,fp8,0,2.4575439453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,4,128,1,bfloat16,bfloat16,0,1.204470443725586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,4,128,1,bfloat16,fp8,0,1.2217344284057616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,4,128,1,fp8,fp8,0,1.1794943809509277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,8,128,1,bfloat16,bfloat16,0,1.3167280197143554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,40,128,1,bfloat16,fp8,0,0.6254447937011719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,40,128,1,fp8,fp8,0,0.6695551872253418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,40,128,1,bfloat16,bfloat16,0,0.7083343982696533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,8,128,1,fp8,fp8,0,1.1265456199645996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,1,128,1,bfloat16,fp8,0,0.5812528133392334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,2,128,1,bfloat16,bfloat16,0,0.603659200668335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,1,128,1,fp8,fp8,0,0.6177279949188232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,2,128,1,bfloat16,fp8,0,0.5801296234130859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,2,128,1,fp8,fp8,0,0.5694719791412354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,4,128,1,bfloat16,bfloat16,0,0.6015744209289551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,4,128,1,bfloat16,fp8,0,0.5793488025665283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,4,128,1,fp8,fp8,0,0.5820799827575683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,8,128,1,bfloat16,fp8,0,0.5742271900177002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,40,128,1,bfloat16,fp8,0,0.3265647888183594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,8,128,1,fp8,fp8,0,0.5794479846954346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,40,128,1,fp8,fp8,0,0.3379551887512207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,1,128,1,bfloat16,bfloat16,0,0.3008847951889038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,1,128,1,bfloat16,fp8,0,0.28950560092926025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,1,128,1,fp8,fp8,0,0.28344800472259524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,2,128,1,bfloat16,fp8,0,0.2814671993255615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,2,128,1,bfloat16,bfloat16,0,0.3168576002120972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,4,128,1,bfloat16,bfloat16,0,0.3045151948928833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,4,128,1,bfloat16,fp8,0,0.2837264060974121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,4,128,1,fp8,fp8,0,0.30311999320983884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,8,128,1,bfloat16,bfloat16,0,0.3078943967819214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,8,128,1,bfloat16,fp8,0,0.28720319271087646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,8,128,1,fp8,fp8,0,0.30652799606323244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,40,8,128,1,bfloat16,fp8,0,1.1871040344238282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,1,128,1,bfloat16,bfloat16,0,0.6235727787017822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,40,8,128,1,bfloat16,bfloat16,0,0.6199456214904785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,2,128,1,fp8,fp8,0,0.29121439456939696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,1,128,1,fp8,fp8,0,5.299571228027344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,1,128,1,bfloat16,fp8,0,5.6279857635498045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,1,128,1,bfloat16,bfloat16,0,6.026460647583008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,40,40,128,1,bfloat16,bfloat16,0,0.36092000007629393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,2,128,1,bfloat16,bfloat16,0,5.889230346679687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,2,128,1,bfloat16,fp8,0,5.261988830566406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,2,128,1,fp8,fp8,0,5.2092735290527346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,4,128,1,bfloat16,fp8,0,5.4048816680908205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,4,128,1,fp8,fp8,0,5.261921691894531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,40,128,1,bfloat16,bfloat16,0,3.5888015747070314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,40,128,1,bfloat16,fp8,0,2.9981855392456054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,8,128,1,bfloat16,fp8,0,5.378432083129883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,4,128,1,bfloat16,bfloat16,0,6.332710266113281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,8,128,1,bfloat16,bfloat16,0,6.252119827270508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,40,8,128,1,fp8,fp8,0,5.34990234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,1,128,1,bfloat16,bfloat16,0,2.8456239700317383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,1,128,1,fp8,fp8,0,2.5912879943847655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,1,128,1,bfloat16,fp8,0,2.77194881439209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,40,128,1,fp8,fp8,0,2.999083137512207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,2,128,1,bfloat16,fp8,0,2.6870752334594727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,2,128,1,bfloat16,bfloat16,0,2.958132743835449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,2,128,1,fp8,fp8,0,2.718262481689453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,4,128,1,bfloat16,bfloat16,0,2.906398391723633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,40,128,1,bfloat16,fp8,0,1.5055919647216798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,40,128,1,bfloat16,bfloat16,0,1.8514287948608399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,4,128,1,fp8,fp8,0,2.636489677429199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,4,128,1,bfloat16,fp8,0,2.887950325012207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,8,128,1,bfloat16,fp8,0,2.6760671615600584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,8,128,1,bfloat16,bfloat16,0,2.9294095993041993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,40,8,128,1,fp8,fp8,0,2.6892223358154297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,40,128,1,fp8,fp8,0,1.709280014038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,1,128,1,bfloat16,bfloat16,0,1.4231887817382813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,1,128,1,bfloat16,fp8,0,1.409489631652832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,1,128,1,fp8,fp8,0,1.2999967575073241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,2,128,1,bfloat16,bfloat16,0,1.4386272430419922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,2,128,1,bfloat16,fp8,0,1.312183952331543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,4,128,1,bfloat16,fp8,0,1.3940896034240722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,2,128,1,fp8,fp8,0,1.526460838317871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,40,128,1,bfloat16,bfloat16,0,0.879635238647461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,4,128,1,fp8,fp8,0,1.4305536270141601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,40,128,1,bfloat16,fp8,0,0.7766272068023682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,8,128,1,bfloat16,fp8,0,1.3400704383850097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,40,128,1,fp8,fp8,0,0.7906879901885986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,8,128,1,bfloat16,bfloat16,0,1.4593824386596679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,8,128,1,fp8,fp8,0,1.4396575927734374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,1,128,1,bfloat16,bfloat16,0,0.7401040077209473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,1,128,1,bfloat16,fp8,0,0.6899903774261474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,1,128,1,fp8,fp8,0,0.6591936111450195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,2,128,1,bfloat16,bfloat16,0,0.7010975837707519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,2,128,1,bfloat16,fp8,0,0.6859280109405518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,2,128,1,fp8,fp8,0,0.7272751808166504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,4,128,1,bfloat16,bfloat16,0,0.7202735900878906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,40,128,1,bfloat16,bfloat16,0,0.4555391788482666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,4,128,1,bfloat16,fp8,0,0.7043360233306885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,4,128,1,fp8,fp8,0,0.6832655906677246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,8,128,1,bfloat16,fp8,0,0.6888144016265869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,8,128,1,fp8,fp8,0,0.6914159774780273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,40,128,1,fp8,fp8,0,0.3930432081222534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,1,128,1,bfloat16,fp8,0,0.33509440422058107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,2,128,1,bfloat16,bfloat16,0,0.3523360013961792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,1,128,1,fp8,fp8,0,0.3541599988937378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,2,128,1,fp8,fp8,0,0.33703999519348143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,4,128,1,bfloat16,bfloat16,0,0.3571023941040039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,4,128,1,bfloat16,fp8,0,0.35470240116119384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,40,4,128,1,bfloat16,bfloat16,0,1.4475199699401855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,4,128,1,fp8,fp8,0,0.3399983882904053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,8,128,1,bfloat16,bfloat16,0,0.36264801025390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,8,128,1,bfloat16,fp8,0,0.3544863939285278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,40,128,1,bfloat16,bfloat16,0,0.23061919212341309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,40,128,1,bfloat16,fp8,0,0.19873759746551514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,40,128,1,fp8,fp8,0,0.2035072088241577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,1,128,1,bfloat16,bfloat16,0,0.18233920335769654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,1,128,1,bfloat16,fp8,0,0.17380000352859498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,1,128,1,fp8,fp8,0,0.17732479572296142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,2,128,1,bfloat16,bfloat16,0,0.1834496021270752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,2,128,1,bfloat16,fp8,0,0.17404639720916748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,2,128,1,fp8,fp8,0,0.17785120010375977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,4,128,1,bfloat16,bfloat16,0,0.1854143977165222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,4,128,1,bfloat16,fp8,0,0.1755328059196472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,4,128,1,fp8,fp8,0,0.17938560247421265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,8,128,1,bfloat16,bfloat16,0,0.18822879791259767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,8,128,1,bfloat16,fp8,0,0.1779855966567993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,40,8,128,1,fp8,fp8,0,0.18174560070037843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,40,8,128,1,bfloat16,bfloat16,0,0.7173120021820069
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,40,128,1,bfloat16,fp8,0,0.418287992477417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,1,128,1,bfloat16,bfloat16,0,0.3624752044677734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,2,128,1,bfloat16,fp8,0,0.3474287986755371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,40,8,128,1,fp8,fp8,0,0.35188000202178954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,1,128,1,bfloat16,fp8,0,5.160521697998047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,1,128,1,fp8,fp8,0,5.179487991333008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,1,128,1,bfloat16,bfloat16,0,5.763695907592774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,2,128,1,bfloat16,fp8,0,5.172577667236328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,2,128,1,fp8,fp8,0,5.125788879394531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,2,128,1,bfloat16,bfloat16,0,5.846001434326172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,4,128,1,bfloat16,bfloat16,0,5.995201492309571
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,4,128,1,bfloat16,fp8,0,5.231484985351562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,40,128,1,bfloat16,fp8,0,3.110977554321289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,40,128,1,bfloat16,bfloat16,0,3.6068256378173826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,4,128,1,fp8,fp8,0,5.180996704101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,1,128,1,bfloat16,bfloat16,0,2.8165008544921877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,8,128,1,bfloat16,fp8,0,5.2969615936279295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,40,128,1,fp8,fp8,0,3.078945541381836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,8,128,1,bfloat16,bfloat16,0,6.2259361267089846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,40,8,128,1,fp8,fp8,0,5.282988739013672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,1,128,1,fp8,fp8,0,2.531398391723633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,1,128,1,bfloat16,fp8,0,2.6872703552246096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,2,128,1,bfloat16,fp8,0,2.5587135314941407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,2,128,1,fp8,fp8,0,2.561404800415039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,2,128,1,bfloat16,bfloat16,0,3.008497619628906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,4,128,1,bfloat16,fp8,0,2.5893743515014647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,4,128,1,bfloat16,bfloat16,0,2.9435279846191404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,4,128,1,fp8,fp8,0,2.582382392883301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,40,128,1,bfloat16,bfloat16,0,1.8341136932373048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,40,128,1,bfloat16,fp8,0,1.5340255737304687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,8,128,1,bfloat16,bfloat16,0,2.9719568252563477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,40,128,1,fp8,fp8,0,1.598051166534424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,8,128,1,bfloat16,fp8,0,2.6548320770263674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,40,8,128,1,fp8,fp8,0,2.6533792495727537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,1,128,1,bfloat16,bfloat16,0,1.3275775909423828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,1,128,1,fp8,fp8,0,1.269331169128418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,1,128,1,bfloat16,fp8,0,1.3396448135375976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,2,128,1,bfloat16,fp8,0,1.2894495964050292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,2,128,1,fp8,fp8,0,1.289572811126709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,4,128,1,bfloat16,bfloat16,0,1.4972384452819825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,4,128,1,bfloat16,fp8,0,1.3767824172973633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,4,128,1,fp8,fp8,0,1.29312162399292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,40,128,1,bfloat16,bfloat16,0,0.899846363067627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,8,128,1,bfloat16,bfloat16,0,1.4702303886413575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,40,128,1,bfloat16,fp8,0,0.822481632232666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,8,128,1,fp8,fp8,0,1.3280719757080077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,8,128,1,bfloat16,fp8,0,1.3370256423950195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,1,128,1,bfloat16,bfloat16,0,0.6720047950744629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,1,128,1,bfloat16,fp8,0,0.6895535945892334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,40,128,1,fp8,fp8,0,0.8079968452453613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,1,128,1,fp8,fp8,0,0.7292448043823242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,2,128,1,bfloat16,fp8,0,0.6513376235961914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,2,128,1,fp8,fp8,0,0.6527599811553955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,4,128,1,bfloat16,bfloat16,0,0.7323887825012207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,4,128,1,bfloat16,fp8,0,0.6908383846282959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,4,128,1,fp8,fp8,0,0.6792191982269287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,8,128,1,bfloat16,bfloat16,0,0.7546991825103759
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,8,128,1,bfloat16,fp8,0,0.6700479984283447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,8,128,1,fp8,fp8,0,0.6746448040008545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,40,128,1,bfloat16,fp8,0,0.4050144195556641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,40,128,1,bfloat16,bfloat16,0,0.47075519561767576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,40,128,1,fp8,fp8,0,0.4060880184173584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,1,128,1,bfloat16,bfloat16,0,0.3609663963317871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,1,128,1,bfloat16,fp8,0,0.3284559965133667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,1,128,1,fp8,fp8,0,0.33001759052276614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,2,128,1,bfloat16,fp8,0,0.35121760368347166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,2,128,1,fp8,fp8,0,0.3375375986099243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,2,128,1,bfloat16,bfloat16,0,0.3528032064437866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,40,2,128,1,bfloat16,bfloat16,0,1.4623488426208495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,4,128,1,bfloat16,bfloat16,0,0.36791200637817384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,4,128,1,bfloat16,fp8,0,0.3342895984649658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,4,128,1,fp8,fp8,0,0.33582239151000975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,8,128,1,bfloat16,bfloat16,0,0.3786688089370728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,8,128,1,bfloat16,fp8,0,0.348855996131897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,40,8,128,1,fp8,fp8,0,0.3487855911254883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,40,128,1,bfloat16,fp8,0,0.21571359634399415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,1,128,1,bfloat16,bfloat16,0,0.17683520317077636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,40,128,1,fp8,fp8,0,0.2056351900100708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,1,128,1,bfloat16,fp8,0,0.17629120349884034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,1,128,1,fp8,fp8,0,0.1736448049545288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,2,128,1,bfloat16,bfloat16,0,0.1820672035217285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,2,128,1,bfloat16,fp8,0,0.17724159955978394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,2,128,1,fp8,fp8,0,0.17125600576400757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,4,128,1,bfloat16,fp8,0,0.17893279790878297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,4,128,1,fp8,fp8,0,0.17689119577407836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,8,128,1,bfloat16,bfloat16,0,0.19092320203781127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,8,128,1,bfloat16,fp8,0,0.18372639417648315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,8,128,1,fp8,fp8,0,0.1768447995185852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,40,128,1,bfloat16,bfloat16,0,0.13256319761276245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,40,128,1,bfloat16,fp8,0,0.1125216007232666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,1,128,1,bfloat16,bfloat16,0,0.09769920110702515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,1,128,1,bfloat16,fp8,0,0.0875536024570465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,1,128,1,fp8,fp8,0,0.0899728000164032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,2,128,1,bfloat16,bfloat16,0,0.0946016013622284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,2,128,1,bfloat16,fp8,0,0.09100480079650879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,2,128,1,fp8,fp8,0,0.08805599808692932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,4,128,1,bfloat16,bfloat16,0,0.09909759759902954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,4,128,1,bfloat16,fp8,0,0.08960000276565552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,4,128,1,fp8,fp8,0,0.09181119799613953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,8,128,1,bfloat16,bfloat16,0,0.09927200078964234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,8,128,1,bfloat16,fp8,0,0.09535359740257263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,8,128,1,fp8,fp8,0,0.09278240203857421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,40,2,128,1,bfloat16,bfloat16,0,0.735862398147583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,40,128,1,bfloat16,bfloat16,0,0.23745760917663575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,40,4,128,1,bfloat16,bfloat16,0,0.1807279944419861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,40,40,128,1,fp8,fp8,0,0.11465920209884643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,1,128,1,bfloat16,fp8,0,3.169296073913574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,1,128,1,fp8,fp8,0,3.213739013671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,1,128,1,bfloat16,bfloat16,0,3.470003128051758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,2,128,1,bfloat16,bfloat16,0,3.4725902557373045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,2,128,1,bfloat16,fp8,0,3.192006492614746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,2,128,1,fp8,fp8,0,3.189726448059082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,40,128,1,bfloat16,bfloat16,0,2.3139135360717775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,4,128,1,bfloat16,bfloat16,0,3.5679393768310548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,4,128,1,bfloat16,fp8,0,3.26834716796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,4,128,1,fp8,fp8,0,3.2149726867675783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,8,128,1,bfloat16,bfloat16,0,3.555780792236328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,8,128,1,bfloat16,fp8,0,3.378121566772461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,40,8,128,1,fp8,fp8,0,3.388896179199219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,40,128,1,bfloat16,fp8,0,1.9820655822753905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,1,128,1,bfloat16,fp8,0,1.5803407669067382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,40,128,1,fp8,fp8,0,1.989313507080078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,1,128,1,fp8,fp8,0,1.6879919052124024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,2,128,1,bfloat16,bfloat16,0,1.6901119232177735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,2,128,1,bfloat16,fp8,0,1.5870623588562012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,2,128,1,fp8,fp8,0,1.5797391891479493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,4,128,1,bfloat16,bfloat16,0,1.742300796508789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,4,128,1,bfloat16,fp8,0,1.611537551879883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,4,128,1,fp8,fp8,0,1.6566783905029296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,8,128,1,bfloat16,fp8,0,1.6431888580322265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,8,128,1,bfloat16,bfloat16,0,1.9337648391723632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,8,128,1,fp8,fp8,0,1.6387199401855468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,40,128,1,bfloat16,fp8,0,0.9997056007385254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,1,128,1,bfloat16,bfloat16,0,0.8161711692810059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,40,128,1,bfloat16,bfloat16,0,1.2074288368225097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,40,128,1,fp8,fp8,0,1.0055232048034668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,1,128,1,bfloat16,fp8,0,0.8760047912597656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,40,1,128,1,bfloat16,bfloat16,0,1.6277280807495118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,1,128,1,fp8,fp8,0,0.788489580154419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,2,128,1,bfloat16,bfloat16,0,0.8315216064453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,2,128,1,bfloat16,fp8,0,0.8007984161376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,4,128,1,bfloat16,bfloat16,0,0.8461055755615234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,2,128,1,fp8,fp8,0,0.8447327613830566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,4,128,1,bfloat16,fp8,0,0.8475296020507812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,4,128,1,fp8,fp8,0,0.8316287994384766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,8,128,1,bfloat16,fp8,0,0.8261887550354003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,40,128,1,bfloat16,bfloat16,0,0.5776063919067382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,8,128,1,bfloat16,bfloat16,0,0.8927136421203613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,40,128,1,bfloat16,fp8,0,0.5132912158966064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,40,8,128,1,fp8,fp8,0,0.8333215713500977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,40,128,1,fp8,fp8,0,0.5223104000091553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,1,128,1,bfloat16,bfloat16,0,0.4239183902740479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,1,128,1,bfloat16,fp8,0,0.4130767822265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,1,128,1,fp8,fp8,0,0.4006783962249756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,2,128,1,bfloat16,bfloat16,0,0.42978081703186033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,2,128,1,bfloat16,fp8,0,0.414521598815918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,2,128,1,fp8,fp8,0,0.41258559226989744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,4,128,1,bfloat16,fp8,0,0.4234799861907959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,4,128,1,fp8,fp8,0,0.41220641136169434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,8,128,1,bfloat16,bfloat16,0,0.45655999183654783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,8,128,1,bfloat16,fp8,0,0.4197103977203369
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,40,128,1,bfloat16,bfloat16,0,0.3040047883987427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,8,128,1,fp8,fp8,0,0.4311312198638916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,40,128,1,bfloat16,fp8,0,0.2659712076187134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,40,128,1,fp8,fp8,0,0.26752800941467286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,1,128,1,bfloat16,fp8,0,0.21258881092071533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,1,128,1,fp8,fp8,0,0.20640640258789061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,2,128,1,bfloat16,bfloat16,0,0.22016639709472657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,2,128,1,fp8,fp8,0,0.21651360988616944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,4,128,1,bfloat16,bfloat16,0,0.22561919689178467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,4,128,1,bfloat16,fp8,0,0.2172976016998291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,4,128,1,fp8,fp8,0,0.21114881038665773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,8,128,1,bfloat16,bfloat16,0,0.23284800052642823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,8,128,1,bfloat16,fp8,0,0.22533121109008789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,40,128,1,bfloat16,bfloat16,0,0.1604367971420288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,8,128,1,fp8,fp8,0,0.22223999500274658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,40,128,1,fp8,fp8,0,0.14222400188446044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,1,128,1,bfloat16,bfloat16,0,0.11708320379257202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,1,128,1,bfloat16,fp8,0,0.112391996383667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,1,128,1,fp8,fp8,0,0.11249439716339112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,2,128,1,bfloat16,bfloat16,0,0.11738400459289551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,2,128,1,bfloat16,fp8,0,0.11431360244750977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,2,128,1,fp8,fp8,0,0.11366560459136962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,4,128,1,bfloat16,bfloat16,0,0.1190384030342102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,4,128,1,bfloat16,fp8,0,0.11484800577163697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,4,128,1,fp8,fp8,0,0.11645439863204957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,8,128,1,bfloat16,fp8,0,0.11741280555725098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,8,128,1,fp8,fp8,0,0.11777280569076538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,40,128,1,bfloat16,bfloat16,0,0.08960319757461548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,40,128,1,bfloat16,fp8,0,0.08069120049476623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,40,128,1,fp8,fp8,0,0.08049920201301575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,1,128,1,bfloat16,fp8,0,0.058955198526382445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,1,128,1,fp8,fp8,0,0.05923200249671936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,2,128,1,bfloat16,bfloat16,0,0.06106399893760681
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,2,128,1,bfloat16,fp8,0,0.05916159749031067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,2,128,1,fp8,fp8,0,0.05918239951133728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,4,128,1,bfloat16,bfloat16,0,0.06380640268325806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,4,128,1,bfloat16,fp8,0,0.05957760214805603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,4,128,1,fp8,fp8,0,0.059628802537918094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,8,128,1,bfloat16,bfloat16,0,0.0678160011768341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,8,128,1,bfloat16,fp8,0,0.060736000537872314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,8,128,1,fp8,fp8,0,0.060540801286697386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,40,4,128,1,bfloat16,bfloat16,0,0.4284992218017578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,1,128,1,bfloat16,bfloat16,0,0.21620960235595704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,40,2,128,1,bfloat16,fp8,0,0.21339199542999268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,40,128,1,bfloat16,fp8,0,0.13853600025177001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,40,8,128,1,bfloat16,bfloat16,0,0.1230623960494995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,40,1,128,1,bfloat16,bfloat16,0,0.0608959972858429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,1,128,1,bfloat16,fp8,0,3.3520030975341797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,1,128,1,bfloat16,bfloat16,0,3.5059967041015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,1,128,1,fp8,fp8,0,3.3715248107910156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,2,128,1,bfloat16,bfloat16,0,3.5111759185791014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,2,128,1,bfloat16,fp8,0,3.364147186279297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,2,128,1,fp8,fp8,0,3.3229103088378906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,4,128,1,bfloat16,bfloat16,0,3.6033710479736327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,4,128,1,bfloat16,fp8,0,3.3991809844970704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,4,128,1,fp8,fp8,0,3.390670394897461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,8,128,1,bfloat16,bfloat16,0,3.7960769653320314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,8,128,1,bfloat16,fp8,0,3.5098846435546873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,40,128,1,bfloat16,bfloat16,0,2.63308162689209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,40,8,128,1,fp8,fp8,0,3.5246448516845703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,40,128,1,bfloat16,fp8,0,2.2378320693969727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,40,128,1,fp8,fp8,0,2.2115936279296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,1,128,1,bfloat16,fp8,0,1.762593650817871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,1,128,1,fp8,fp8,0,1.653856086730957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,2,128,1,bfloat16,bfloat16,0,1.7518863677978516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,2,128,1,bfloat16,fp8,0,1.668649673461914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,2,128,1,fp8,fp8,0,1.6631935119628907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,4,128,1,bfloat16,bfloat16,0,1.7355295181274415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,4,128,1,bfloat16,fp8,0,1.735860824584961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,4,128,1,fp8,fp8,0,1.6969007492065429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,8,128,1,bfloat16,bfloat16,0,1.9088560104370118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,40,128,1,bfloat16,bfloat16,0,1.3135343551635743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,8,128,1,bfloat16,fp8,0,1.7636495590209962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,1,128,1,bfloat16,bfloat16,0,1.6675567626953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,40,128,1,bfloat16,fp8,0,1.119878387451172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,40,8,128,1,fp8,fp8,0,1.7571632385253906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,1,128,1,bfloat16,bfloat16,0,0.8441167831420898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,40,128,1,fp8,fp8,0,1.1222960472106933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,1,128,1,bfloat16,fp8,0,0.8877264022827148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,2,128,1,bfloat16,bfloat16,0,0.8592127799987793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,1,128,1,fp8,fp8,0,0.8583904266357422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,2,128,1,fp8,fp8,0,0.8418335914611816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,4,128,1,bfloat16,fp8,0,0.8630640029907226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,4,128,1,fp8,fp8,0,0.8694911956787109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,40,128,1,bfloat16,bfloat16,0,0.6649072170257568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,8,128,1,bfloat16,fp8,0,0.888582420349121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,40,128,1,bfloat16,fp8,0,0.5651584148406983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,8,128,1,fp8,fp8,0,0.910153579711914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,40,128,1,fp8,fp8,0,0.568723201751709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,1,128,1,bfloat16,bfloat16,0,0.42859840393066406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,1,128,1,bfloat16,fp8,0,0.4331520080566406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,1,128,1,fp8,fp8,0,0.4248239994049072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,2,128,1,bfloat16,bfloat16,0,0.44683837890625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,2,128,1,bfloat16,fp8,0,0.4325839996337891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,2,128,1,fp8,fp8,0,0.42737441062927245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,4,128,1,bfloat16,bfloat16,0,0.4561168193817139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,4,128,1,bfloat16,fp8,0,0.43927202224731443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,4,128,1,fp8,fp8,0,0.4444176197052002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,8,128,1,bfloat16,bfloat16,0,0.466537618637085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,8,128,1,bfloat16,fp8,0,0.45572800636291505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,40,128,1,bfloat16,fp8,0,0.29483039379119874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,40,128,1,bfloat16,bfloat16,0,0.34617919921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,40,8,128,1,fp8,fp8,0,0.45953922271728515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,40,128,1,fp8,fp8,0,0.2899679899215698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,1,128,1,bfloat16,bfloat16,0,0.22640480995178222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,1,128,1,bfloat16,fp8,0,0.22307360172271729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,2,128,1,bfloat16,bfloat16,0,0.22966558933258058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,2,128,1,bfloat16,fp8,0,0.22002239227294923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,2,128,1,fp8,fp8,0,0.22444159984588624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,4,128,1,bfloat16,bfloat16,0,0.23519039154052734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,4,128,1,bfloat16,fp8,0,0.2288111925125122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,4,128,1,fp8,fp8,0,0.2244256019592285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,8,128,1,bfloat16,bfloat16,0,0.24560320377349854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,8,128,1,bfloat16,fp8,0,0.2373136043548584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,40,128,1,bfloat16,bfloat16,0,0.17933280467987062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,8,128,1,fp8,fp8,0,0.23682079315185547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,40,128,1,bfloat16,fp8,0,0.1564144015312195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,1,128,1,bfloat16,bfloat16,0,0.11722240447998047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,1,128,1,bfloat16,fp8,0,0.1180832028388977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,40,128,1,fp8,fp8,0,0.15661760568618774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,1,128,1,fp8,fp8,0,0.1183616042137146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,2,128,1,bfloat16,bfloat16,0,0.1186303973197937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,2,128,1,fp8,fp8,0,0.11924959421157837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,4,128,1,bfloat16,bfloat16,0,0.12421760559082032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,4,128,1,bfloat16,fp8,0,0.11889920234680176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,4,128,1,fp8,fp8,0,0.12123359441757202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,8,128,1,bfloat16,bfloat16,0,0.12997280359268187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,8,128,1,bfloat16,bfloat16,0,0.9311599731445312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,8,128,1,bfloat16,fp8,0,0.12228319644927979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,40,128,1,bfloat16,bfloat16,0,0.09908480048179627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,8,128,1,fp8,fp8,0,0.1248128056526184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,40,128,1,bfloat16,fp8,0,0.08593760132789612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,1,128,1,bfloat16,bfloat16,0,0.06675519943237304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,40,128,1,fp8,fp8,0,0.08624799847602845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,1,128,1,fp8,fp8,0,0.06172800064086914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,2,128,1,bfloat16,bfloat16,0,0.0671392023563385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,2,128,1,bfloat16,fp8,0,0.06208800077438355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,2,128,1,fp8,fp8,0,0.06190239787101746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,4,128,1,bfloat16,bfloat16,0,0.0695360004901886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,4,128,1,bfloat16,fp8,0,0.06486080288887024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,4,128,1,fp8,fp8,0,0.0639631986618042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,8,128,1,bfloat16,bfloat16,0,0.07136480212211609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,8,128,1,bfloat16,fp8,0,0.0675055980682373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,8,128,1,fp8,fp8,0,0.06714239716529846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,40,128,1,bfloat16,bfloat16,0,0.05886560082435608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,40,128,1,bfloat16,fp8,0,0.0498991996049881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,40,128,1,fp8,fp8,0,0.04970560073852539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,1,128,1,bfloat16,bfloat16,0,0.036934399604797365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,1,128,1,fp8,fp8,0,0.036180800199508666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,2,128,1,bfloat16,bfloat16,0,0.037385600805282596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,2,128,1,bfloat16,fp8,0,0.03622080087661743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,2,128,1,fp8,fp8,0,0.03707680106163025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,4,128,1,bfloat16,bfloat16,0,0.03665919899940491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,4,128,1,bfloat16,fp8,0,0.036687999963760376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,4,128,1,fp8,fp8,0,0.036559998989105225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,8,128,1,bfloat16,bfloat16,0,0.03760960102081299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,8,128,1,bfloat16,fp8,0,0.03702239990234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,8,128,1,fp8,fp8,0,0.0368912011384964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,40,1,128,1,fp8,fp8,0,0.22178239822387696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,2,128,1,bfloat16,fp8,0,0.85720796585083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,40,4,128,1,bfloat16,bfloat16,0,0.8749888420104981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,40,2,128,1,bfloat16,fp8,0,0.11868000030517578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,40,1,128,1,bfloat16,fp8,0,0.06247519850730896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,1,128,1,bfloat16,bfloat16,0,2.403532791137695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,1,128,1,bfloat16,fp8,0,2.420305633544922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,40,1,128,1,bfloat16,fp8,0,0.03671680092811584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,1,128,1,fp8,fp8,0,2.4056112289428713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,2,128,1,bfloat16,bfloat16,0,2.4823808670043945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,2,128,1,bfloat16,fp8,0,2.452553558349609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,2,128,1,fp8,fp8,0,2.4235183715820314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,4,128,1,bfloat16,bfloat16,0,2.5756704330444338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,4,128,1,bfloat16,fp8,0,2.4835088729858397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,4,128,1,fp8,fp8,0,2.502769660949707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,8,128,1,bfloat16,bfloat16,0,2.768107223510742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,8,128,1,bfloat16,fp8,0,2.6294336318969727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,1,128,1,bfloat16,bfloat16,0,1.2051759719848634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,1,128,1,bfloat16,fp8,0,1.1976688385009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,40,128,1,bfloat16,fp8,0,1.7809408187866211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,40,128,1,fp8,fp8,0,1.772318458557129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,1,128,1,fp8,fp8,0,1.2050800323486328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,2,128,1,bfloat16,bfloat16,0,1.2355504035949707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,2,128,1,bfloat16,fp8,0,1.2077743530273437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,2,128,1,fp8,fp8,0,1.2052160263061524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,4,128,1,bfloat16,bfloat16,0,1.291262435913086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,4,128,1,bfloat16,fp8,0,1.2370736122131347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,4,128,1,fp8,fp8,0,1.2360159873962402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,8,128,1,bfloat16,bfloat16,0,1.3918911933898925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,8,128,1,fp8,fp8,0,1.3093631744384766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,8,128,1,bfloat16,fp8,0,1.3143247604370116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,40,128,1,bfloat16,bfloat16,0,1.0622847557067872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,40,128,1,bfloat16,fp8,0,0.887600040435791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,40,8,128,1,fp8,fp8,0,2.636854362487793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,40,128,1,fp8,fp8,0,0.8873135566711425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,1,128,1,bfloat16,fp8,0,0.6028543949127197
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,1,128,1,bfloat16,bfloat16,0,0.6301599979400635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,40,40,128,1,bfloat16,bfloat16,0,2.103156852722168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,2,128,1,bfloat16,fp8,0,0.6082848072052002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,2,128,1,bfloat16,bfloat16,0,0.6298064231872559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,4,128,1,bfloat16,bfloat16,0,0.6466896057128906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,2,128,1,fp8,fp8,0,0.6134143829345703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,4,128,1,fp8,fp8,0,0.6461391925811768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,8,128,1,bfloat16,fp8,0,0.6603456020355225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,8,128,1,fp8,fp8,0,0.6719215869903564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,8,128,1,bfloat16,bfloat16,0,0.703710412979126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,1,128,1,bfloat16,bfloat16,0,0.3098047971725464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,1,128,1,bfloat16,fp8,0,0.31422879695892336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,40,128,1,fp8,fp8,0,0.4612304210662842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,40,128,1,bfloat16,bfloat16,0,0.5380544185638427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,1,128,1,fp8,fp8,0,0.3109776020050049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,2,128,1,bfloat16,bfloat16,0,0.317303991317749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,2,128,1,bfloat16,fp8,0,0.3164959907531738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,2,128,1,fp8,fp8,0,0.3191535949707031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,4,128,1,bfloat16,fp8,0,0.3227936029434204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,4,128,1,bfloat16,bfloat16,0,0.33730080127716067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,4,128,1,fp8,fp8,0,0.3198544025421143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,8,128,1,bfloat16,bfloat16,0,0.3610944032669067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,8,128,1,bfloat16,fp8,0,0.3458303928375244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,40,128,1,bfloat16,bfloat16,0,0.2807823896408081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,40,128,1,bfloat16,fp8,0,0.23396000862121583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,1,128,1,bfloat16,fp8,0,0.16478079557418823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,1,128,1,fp8,fp8,0,0.16025439500808716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,2,128,1,bfloat16,bfloat16,0,0.1684928059577942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,2,128,1,bfloat16,fp8,0,0.16165599822998047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,2,128,1,fp8,fp8,0,0.16637279987335205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,4,128,1,bfloat16,bfloat16,0,0.17197599411010742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,4,128,1,bfloat16,fp8,0,0.17014240026473998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,1,128,1,fp8,fp8,0,0.6067567825317383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,4,128,1,fp8,fp8,0,0.165665602684021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,8,128,1,bfloat16,bfloat16,0,0.18762719631195068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,8,128,1,bfloat16,fp8,0,0.1764847993850708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,8,128,1,fp8,fp8,0,0.17524319887161255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,40,128,1,bfloat16,bfloat16,0,0.14822080135345458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,40,128,1,bfloat16,fp8,0,0.12581599950790406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,40,128,1,fp8,fp8,0,0.12519840002059937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,1,128,1,bfloat16,bfloat16,0,0.09012960195541382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,1,128,1,bfloat16,fp8,0,0.08733119964599609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,40,4,128,1,bfloat16,fp8,0,0.625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,1,128,1,fp8,fp8,0,0.08678399920463561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,2,128,1,bfloat16,bfloat16,0,0.09164320230484009
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,2,128,1,bfloat16,fp8,0,0.08824959993362427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,2,128,1,fp8,fp8,0,0.08739200234413147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,4,128,1,bfloat16,bfloat16,0,0.09159359931945801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,4,128,1,bfloat16,fp8,0,0.09211999773979188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,4,128,1,fp8,fp8,0,0.09020000100135803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,8,128,1,bfloat16,bfloat16,0,0.09808639883995056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,8,128,1,bfloat16,fp8,0,0.09368799924850464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,40,8,128,1,fp8,fp8,0,0.09620320200920104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,40,128,1,bfloat16,bfloat16,0,0.08076480031013489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,40,128,1,bfloat16,fp8,0,0.06949440240859986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,40,128,1,bfloat16,fp8,0,0.4553328037261963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,40,128,1,fp8,fp8,0,0.06995999813079834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,2,128,1,bfloat16,bfloat16,0,0.05067520141601563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,2,128,1,bfloat16,fp8,0,0.046865600347518924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,2,128,1,fp8,fp8,0,0.04702560007572174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,4,128,1,bfloat16,bfloat16,0,0.05199360251426697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,4,128,1,bfloat16,fp8,0,0.048588800430297854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,4,128,1,fp8,fp8,0,0.0483951985836029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,8,128,1,bfloat16,bfloat16,0,0.054497599601745605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,40,8,128,1,fp8,fp8,0,0.3400912046432495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,8,128,1,bfloat16,fp8,0,0.051399999856948854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,8,128,1,fp8,fp8,0,0.052083200216293334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,40,128,1,bfloat16,bfloat16,0,0.046983999013900754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,40,128,1,bfloat16,fp8,0,0.0396912008523941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,40,128,1,fp8,fp8,0,0.23733599185943605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,40,128,1,fp8,fp8,0,0.039483198523521425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,1,128,1,bfloat16,bfloat16,0,0.027982398867607117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,1,128,1,bfloat16,fp8,0,0.029455998539924623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,1,128,1,fp8,fp8,0,0.028585600852966308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,2,128,1,bfloat16,bfloat16,0,0.028123199939727783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,2,128,1,bfloat16,fp8,0,0.02948000133037567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,2,128,1,fp8,fp8,0,0.028431999683380126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,4,128,1,bfloat16,bfloat16,0,0.028435200452804565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,4,128,1,bfloat16,fp8,0,0.029500800371170043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,4,128,1,fp8,fp8,0,0.028591999411582948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,8,128,1,bfloat16,fp8,0,0.029760000109672547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,8,128,1,fp8,fp8,0,0.02898080050945282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,40,128,1,bfloat16,bfloat16,0,0.026096001267433167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,40,128,1,bfloat16,fp8,0,0.025379198789596557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,40,128,1,fp8,fp8,0,0.025905600190162657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,1,128,1,bfloat16,bfloat16,0,0.018772800266742707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,1,128,1,bfloat16,fp8,0,0.020278400182723998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,1,128,1,fp8,fp8,0,0.01955360025167465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,2,128,1,bfloat16,bfloat16,0,0.019351999461650848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,2,128,1,bfloat16,fp8,0,0.019500799477100372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,2,128,1,fp8,fp8,0,0.020257599651813507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,4,128,1,bfloat16,bfloat16,0,0.018838399648666383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,4,128,1,bfloat16,fp8,0,0.020308800041675568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,4,128,1,fp8,fp8,0,0.019675199687480927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,8,128,1,bfloat16,bfloat16,0,0.019913600385189058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,8,128,1,bfloat16,fp8,0,0.02001439929008484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,40,8,128,1,fp8,fp8,0,0.0206496000289917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,1,128,1,bfloat16,bfloat16,0,0.0514415979385376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,1,128,1,fp8,fp8,0,0.04615199863910675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,1,128,1,bfloat16,fp8,0,0.9831999778747559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,1,128,1,fp8,fp8,0,0.9813872337341308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,2,128,1,bfloat16,bfloat16,0,0.9526608467102051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,40,1,128,1,bfloat16,bfloat16,0,0.16548479795455934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,2,128,1,bfloat16,fp8,0,0.990056037902832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,40,8,128,1,bfloat16,bfloat16,0,0.02930560111999512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,2,128,1,fp8,fp8,0,0.9832703590393066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,4,128,1,bfloat16,bfloat16,0,1.0000160217285157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,4,128,1,bfloat16,fp8,0,1.024174404144287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,4,128,1,fp8,fp8,0,1.01692476272583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,8,128,1,bfloat16,bfloat16,0,1.0965824127197266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,40,1,128,1,bfloat16,fp8,0,0.04678240120410919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,8,128,1,bfloat16,fp8,0,1.070359992980957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,40,128,1,bfloat16,bfloat16,0,0.9648015975952149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,40,128,1,bfloat16,fp8,0,0.7552559852600098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,8,128,1,fp8,fp8,0,1.0685855865478515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,40,128,1,fp8,fp8,0,0.7565199851989746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,1,128,1,bfloat16,bfloat16,0,0.4728672027587891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,1,128,1,bfloat16,fp8,0,0.4941728115081787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,2,128,1,bfloat16,bfloat16,0,0.4828479766845703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,1,128,1,fp8,fp8,0,0.49524798393249514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,2,128,1,bfloat16,fp8,0,0.49840641021728516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,2,128,1,fp8,fp8,0,0.49894399642944337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,4,128,1,bfloat16,bfloat16,0,0.5069215774536133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,4,128,1,bfloat16,fp8,0,0.5161168098449707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,4,128,1,fp8,fp8,0,0.5155168056488038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,8,128,1,bfloat16,fp8,0,0.5377359867095948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,8,128,1,bfloat16,bfloat16,0,0.5544479846954345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,40,8,128,1,fp8,fp8,0,0.5418367862701416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,1,128,1,bfloat16,bfloat16,0,0.24139680862426757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,40,128,1,bfloat16,fp8,0,0.385862398147583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,40,128,1,bfloat16,bfloat16,0,0.49008960723876954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,1,128,1,fp8,fp8,0,0.2537519931793213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,2,128,1,bfloat16,fp8,0,0.25430879592895506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,2,128,1,bfloat16,bfloat16,0,0.250766396522522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,4,128,1,bfloat16,bfloat16,0,0.2600831985473633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,4,128,1,bfloat16,fp8,0,0.2646512031555176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,4,128,1,fp8,fp8,0,0.26281280517578126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,8,128,1,bfloat16,bfloat16,0,0.28602240085601804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,8,128,1,bfloat16,fp8,0,0.27758240699768066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,8,128,1,fp8,fp8,0,0.2777487993240356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,40,128,1,bfloat16,bfloat16,0,0.2526144027709961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,40,128,1,bfloat16,fp8,0,0.2014159917831421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,40,128,1,fp8,fp8,0,0.19999040365219117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,1,128,1,bfloat16,fp8,0,0.13184319734573363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,1,128,1,fp8,fp8,0,0.13403359651565552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,2,128,1,bfloat16,bfloat16,0,0.13045599460601806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,2,128,1,bfloat16,fp8,0,0.13343520164489747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,2,128,1,fp8,fp8,0,0.13547359704971312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,4,128,1,bfloat16,bfloat16,0,0.1369647979736328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,4,128,1,bfloat16,fp8,0,0.13741919994354249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,4,128,1,fp8,fp8,0,0.13945759534835817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,8,128,1,bfloat16,bfloat16,0,0.1487104058265686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,8,128,1,bfloat16,fp8,0,0.14434080123901366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,8,128,1,fp8,fp8,0,0.14631359577178954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,40,128,1,bfloat16,bfloat16,0,0.13399840593338014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,40,128,1,bfloat16,fp8,0,0.10650240182876587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,40,128,1,fp8,fp8,0,0.10796639919281006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,1,128,1,bfloat16,bfloat16,0,0.07060319781303406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,1,128,1,bfloat16,fp8,0,0.07191200256347656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,1,128,1,fp8,fp8,0,0.07355359792709351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,2,128,1,bfloat16,bfloat16,0,0.07155680060386657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,2,128,1,bfloat16,fp8,0,0.07442880272865296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,2,128,1,fp8,fp8,0,0.07439360022544861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,4,128,1,bfloat16,bfloat16,0,0.07535039782524108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,4,128,1,bfloat16,fp8,0,0.07656800150871276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,4,128,1,fp8,fp8,0,0.07609440088272094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,8,128,1,bfloat16,bfloat16,0,0.08161600232124329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,8,128,1,bfloat16,fp8,0,0.07908480167388916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,40,8,128,1,fp8,fp8,0,0.07944480180740357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,40,128,1,bfloat16,bfloat16,0,0.07446399927139283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,40,128,1,fp8,fp8,0,0.06128479838371277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,1,128,1,bfloat16,bfloat16,0,0.041910400986671446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,40,128,1,fp8,fp8,0,0.384716796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,1,128,1,bfloat16,fp8,0,0.04056319892406464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,1,128,1,fp8,fp8,0,0.03939040005207062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,2,128,1,bfloat16,bfloat16,0,0.04255360066890716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,2,128,1,bfloat16,fp8,0,0.03996480107307434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,2,128,1,fp8,fp8,0,0.256113600730896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,2,128,1,fp8,fp8,0,0.04076319932937622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,4,128,1,bfloat16,bfloat16,0,0.04345600008964538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,4,128,1,bfloat16,fp8,0,0.041633599996566774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,8,128,1,bfloat16,bfloat16,0,0.045947200059890746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,8,128,1,bfloat16,fp8,0,0.04318560063838959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,8,128,1,fp8,fp8,0,0.04344640076160431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,40,128,1,bfloat16,bfloat16,0,0.04227519929409027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,40,128,1,bfloat16,fp8,0,0.03399359881877899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,40,128,1,fp8,fp8,0,0.0342960000038147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,1,128,1,bfloat16,bfloat16,0,0.023238399624824525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,1,128,1,bfloat16,fp8,0,0.024852800369262695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,1,128,1,fp8,fp8,0,0.024876800179481507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,40,1,128,1,bfloat16,bfloat16,0,0.12771040201187134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,2,128,1,bfloat16,bfloat16,0,0.023396800458431243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,2,128,1,bfloat16,fp8,0,0.025099200010299683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,2,128,1,fp8,fp8,0,0.024803200364112855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,4,128,1,bfloat16,fp8,0,0.025209599733352663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,4,128,1,fp8,fp8,0,0.02510719895362854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,8,128,1,bfloat16,bfloat16,0,0.024934400618076325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,8,128,1,bfloat16,fp8,0,0.025441598892211915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,8,128,1,fp8,fp8,0,0.02550080120563507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,40,128,1,bfloat16,bfloat16,0,0.023214399814605713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,40,128,1,bfloat16,fp8,0,0.021860800683498383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,40,128,1,fp8,fp8,0,0.02187040001153946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,40,1,128,1,bfloat16,bfloat16,0,0.9252400398254395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,1,128,1,bfloat16,bfloat16,0,0.017027199268341064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,1,128,1,bfloat16,fp8,0,0.01812479943037033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,2,128,1,bfloat16,bfloat16,0,0.016947199404239655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,2,128,1,bfloat16,fp8,0,0.018092800676822663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,1,128,1,fp8,fp8,0,0.018615999817848207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,4,128,1,bfloat16,bfloat16,0,0.017543999850749968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,4,128,1,bfloat16,fp8,0,0.018617600202560425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,4,128,1,fp8,fp8,0,0.018115200102329254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,8,128,1,bfloat16,bfloat16,0,0.01808159947395325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,8,128,1,bfloat16,fp8,0,0.01828480064868927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,8,128,1,fp8,fp8,0,0.0188511997461319
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,40,128,1,bfloat16,bfloat16,0,0.01629280000925064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,40,128,1,bfloat16,fp8,0,0.016624000668525696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,40,128,1,fp8,fp8,0,0.01679999977350235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,1,128,1,bfloat16,bfloat16,0,0.014667199552059173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,1,128,1,bfloat16,fp8,0,0.014988799393177033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,1,128,1,fp8,fp8,0,0.015214399993419647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,2,128,1,bfloat16,bfloat16,0,0.014256000518798828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,2,128,1,bfloat16,fp8,0,0.015414400398731232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,2,128,1,fp8,fp8,0,0.014871999621391296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,4,128,1,bfloat16,bfloat16,0,0.014846399426460266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,4,128,1,bfloat16,fp8,0,0.014956800639629364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,4,128,1,fp8,fp8,0,0.015464000403881073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,8,128,1,bfloat16,bfloat16,0,0.014716799557209014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,8,128,1,bfloat16,fp8,0,0.01579200029373169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,40,8,128,1,fp8,fp8,0,0.015160000324249268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,40,128,1,bfloat16,fp8,0,0.060843199491500854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,40,1,128,1,bfloat16,fp8,0,0.25658719539642333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,1,128,1,bfloat16,bfloat16,0,0.4589200019836426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,1,128,1,bfloat16,fp8,0,0.4905280113220215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,1,128,1,fp8,fp8,0,0.48860797882080076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,40,4,128,1,fp8,fp8,0,0.04210880100727081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,2,128,1,bfloat16,bfloat16,0,0.4678463935852051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,2,128,1,bfloat16,fp8,0,0.5017471790313721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,2,128,1,fp8,fp8,0,0.49974560737609863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,4,128,1,bfloat16,bfloat16,0,0.48369598388671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,4,128,1,bfloat16,fp8,0,0.5236911773681641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,4,128,1,fp8,fp8,0,0.5234896183013916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,40,4,128,1,bfloat16,bfloat16,0,0.02393600046634674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,8,128,1,bfloat16,bfloat16,0,0.5250415802001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,8,128,1,bfloat16,fp8,0,0.5584752082824707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,40,8,128,1,fp8,fp8,0,0.5589231967926025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,40,128,1,bfloat16,fp8,0,0.402620792388916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,40,128,1,fp8,fp8,0,0.4007904052734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,1,128,1,bfloat16,bfloat16,0,0.23412160873413085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,1,128,1,bfloat16,fp8,0,0.2522783994674683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,1,128,1,fp8,fp8,0,0.25233280658721924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,40,2,128,1,fp8,fp8,0,0.01809920072555542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,2,128,1,bfloat16,bfloat16,0,0.24185121059417725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,2,128,1,bfloat16,fp8,0,0.2578160047531128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,2,128,1,fp8,fp8,0,0.25611200332641604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,4,128,1,bfloat16,bfloat16,0,0.24796319007873535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,4,128,1,bfloat16,fp8,0,0.2662607908248901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,4,128,1,fp8,fp8,0,0.2664783954620361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,8,128,1,bfloat16,bfloat16,0,0.26843359470367434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,8,128,1,bfloat16,fp8,0,0.2863087892532349
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,40,128,1,bfloat16,bfloat16,0,0.2527551889419556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,8,128,1,fp8,fp8,0,0.2863375902175903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,1,128,1,bfloat16,bfloat16,0,0.12528959512710572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,40,128,1,fp8,fp8,0,0.2076927900314331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,1,128,1,bfloat16,fp8,0,0.12931519746780396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,1,128,1,fp8,fp8,0,0.13297439813613893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,2,128,1,bfloat16,bfloat16,0,0.1263152003288269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,2,128,1,bfloat16,fp8,0,0.1356127977371216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,2,128,1,fp8,fp8,0,0.13528159856796265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,4,128,1,bfloat16,bfloat16,0,0.13037120103836058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,4,128,1,fp8,fp8,0,0.14027199745178223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,8,128,1,bfloat16,bfloat16,0,0.14013439416885376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,8,128,1,bfloat16,fp8,0,0.14838080406188964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,8,128,1,fp8,fp8,0,0.14726719856262208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,40,128,1,bfloat16,bfloat16,0,0.13163520097732545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,1,128,1,bfloat16,bfloat16,0,0.06777920126914978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,40,128,1,fp8,fp8,0,0.10809760093688965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,1,128,1,bfloat16,fp8,0,0.07138080000877381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,1,128,1,fp8,fp8,0,0.07103360295295716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,2,128,1,bfloat16,bfloat16,0,0.0693455994129181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,2,128,1,bfloat16,fp8,0,0.07264320254325866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,2,128,1,fp8,fp8,0,0.07212640047073364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,4,128,1,bfloat16,bfloat16,0,0.07264320254325866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,4,128,1,bfloat16,fp8,0,0.07419360280036927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,4,128,1,fp8,fp8,0,0.07333920001983643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,8,128,1,bfloat16,bfloat16,0,0.07704799771308898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,8,128,1,bfloat16,fp8,0,0.07755680084228515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,8,128,1,fp8,fp8,0,0.07731840014457703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,40,128,1,bfloat16,bfloat16,0,0.07337920069694519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,40,128,1,bfloat16,fp8,0,0.05799040198326111
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,40,128,1,fp8,fp8,0,0.05766400098800659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,1,128,1,bfloat16,fp8,0,0.03586240112781525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,1,128,1,fp8,fp8,0,0.035627201199531555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,2,128,1,bfloat16,bfloat16,0,0.039680001139640805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,2,128,1,bfloat16,fp8,0,0.036078399419784545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,2,128,1,fp8,fp8,0,0.03624959886074066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,4,128,1,bfloat16,bfloat16,0,0.04116640090942383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,4,128,1,bfloat16,fp8,0,0.03788959980010986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,4,128,1,fp8,fp8,0,0.03781760036945343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,8,128,1,bfloat16,bfloat16,0,0.044064000248909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,8,128,1,bfloat16,fp8,0,0.04190880060195923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,8,128,1,fp8,fp8,0,0.04177759885787964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,40,128,1,bfloat16,bfloat16,0,0.041652798652648926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,40,128,1,fp8,fp8,0,0.03069919943809509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,40,40,128,1,bfloat16,bfloat16,0,0.4811295986175537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,1,128,1,bfloat16,bfloat16,0,0.022473600506782532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,1,128,1,bfloat16,fp8,0,0.02390879988670349
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,1,128,1,fp8,fp8,0,0.023918400704860687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,2,128,1,bfloat16,bfloat16,0,0.02253279983997345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,2,128,1,bfloat16,fp8,0,0.023839999735355378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,2,128,1,fp8,fp8,0,0.023902399837970732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,4,128,1,bfloat16,bfloat16,0,0.02276960015296936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,4,128,1,bfloat16,fp8,0,0.02406720072031021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,4,128,1,fp8,fp8,0,0.024033600091934205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,8,128,1,bfloat16,bfloat16,0,0.023790399730205535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,8,128,1,bfloat16,fp8,0,0.024300800263881685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,8,128,1,fp8,fp8,0,0.02463199943304062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,40,128,1,bfloat16,fp8,0,0.20773439407348632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,40,128,1,bfloat16,bfloat16,0,0.021721599996089934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,40,128,1,bfloat16,fp8,0,0.020257599651813507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,1,128,1,bfloat16,bfloat16,0,0.01743520051240921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,1,128,1,bfloat16,fp8,0,0.018940800428390504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,1,128,1,fp8,fp8,0,0.018561600148677825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,2,128,1,bfloat16,bfloat16,0,0.017399999499320983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,2,128,1,bfloat16,fp8,0,0.018537600338459016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,2,128,1,fp8,fp8,0,0.01835840046405792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,4,128,1,bfloat16,bfloat16,0,0.017710399627685548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,4,128,1,bfloat16,fp8,0,0.018401600420475006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,40,4,128,1,bfloat16,fp8,0,0.13948320150375365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,4,128,1,fp8,fp8,0,0.018382400274276733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,8,128,1,bfloat16,bfloat16,0,0.017907199263572694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,8,128,1,bfloat16,fp8,0,0.018718400597572328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,8,128,1,fp8,fp8,0,0.01847359985113144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,40,128,1,bfloat16,fp8,0,0.014670400321483612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,40,128,1,fp8,fp8,0,0.014286400377750396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,1,128,1,bfloat16,bfloat16,0,0.012984000146389008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,1,128,1,bfloat16,fp8,0,0.013195200264453888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,1,128,1,fp8,fp8,0,0.013756799697875976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,2,128,1,bfloat16,bfloat16,0,0.01268640011548996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,2,128,1,bfloat16,fp8,0,0.013548800349235534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,4,128,1,bfloat16,bfloat16,0,0.012943999469280243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,2,128,1,fp8,fp8,0,0.013631999492645264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,4,128,1,fp8,fp8,0,0.013283200562000275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,8,128,1,bfloat16,bfloat16,0,0.013284799456596375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,8,128,1,bfloat16,fp8,0,0.01342719942331314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,8,128,1,fp8,fp8,0,0.013544000685214996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,40,128,1,bfloat16,bfloat16,0,0.013872000575065612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,40,128,1,bfloat16,fp8,0,0.01440960019826889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,40,128,1,fp8,fp8,0,0.014049600064754485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,1,128,1,bfloat16,bfloat16,0,0.012656000256538392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,1,128,1,bfloat16,fp8,0,0.013171200454235078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,40,1,128,1,bfloat16,bfloat16,0,0.03859840035438537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,1,128,1,fp8,fp8,0,0.01329759955406189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,2,128,1,bfloat16,bfloat16,0,0.012521600723266602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,2,128,1,fp8,fp8,0,0.01327040046453476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,2,128,1,bfloat16,fp8,0,0.013259199261665345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,4,128,1,bfloat16,bfloat16,0,0.012654399871826172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,4,128,1,bfloat16,fp8,0,0.013307200372219085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,8,128,1,bfloat16,bfloat16,0,0.012700800597667695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,8,128,1,bfloat16,fp8,0,0.013553600013256072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,8,128,1,fp8,fp8,0,0.013236799836158752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,1,128,1,bfloat16,bfloat16,0,0.23540959358215333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,40,40,128,1,bfloat16,fp8,0,0.03155680000782013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,1,128,1,bfloat16,fp8,0,0.2533200025558472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,2,128,1,bfloat16,bfloat16,0,0.2387295961380005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,40,40,128,1,fp8,fp8,0,0.019948799908161164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,2,128,1,bfloat16,fp8,0,0.25863521099090575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,2,128,1,fp8,fp8,0,0.2556112051010132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,4,128,1,bfloat16,bfloat16,0,0.25007200241088867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,40,128,1,bfloat16,bfloat16,0,0.01491519957780838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,40,40,128,1,bfloat16,fp8,0,0.10818079710006714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,4,128,1,bfloat16,fp8,0,0.26465439796447754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,4,128,1,fp8,fp8,0,0.26674880981445315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,8,128,1,bfloat16,bfloat16,0,0.2988991975784302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,40,4,128,1,bfloat16,fp8,0,0.01356319934129715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,8,128,1,fp8,fp8,0,0.3131711959838867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,40,128,1,bfloat16,bfloat16,0,0.30611999034881593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,40,128,1,bfloat16,fp8,0,0.2920192003250122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,1,128,1,bfloat16,bfloat16,0,0.12341760396957398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,40,128,1,fp8,fp8,0,0.2923680067062378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,1,128,1,bfloat16,fp8,0,0.1333567976951599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,1,128,1,fp8,fp8,0,0.13417600393295287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,2,128,1,bfloat16,bfloat16,0,0.12532320022583007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,2,128,1,bfloat16,fp8,0,0.13453919887542726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,2,128,1,fp8,fp8,0,0.13583199977874755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,4,128,1,bfloat16,bfloat16,0,0.13084640502929687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,40,4,128,1,fp8,fp8,0,0.013233600556850434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,4,128,1,bfloat16,fp8,0,0.1388975977897644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,4,128,1,fp8,fp8,0,0.13930879831314086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,8,128,1,bfloat16,bfloat16,0,0.1559440016746521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,8,128,1,bfloat16,fp8,0,0.16451359987258912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,40,8,128,1,fp8,fp8,0,0.16411679983139038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,40,128,1,bfloat16,bfloat16,0,0.16147520542144775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,40,128,1,bfloat16,fp8,0,0.15357120037078859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,1,128,1,bfloat16,bfloat16,0,0.06764479875564575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,1,128,1,bfloat16,fp8,0,0.07129120230674743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,1,128,1,fp8,fp8,0,0.07140960097312928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,2,128,1,bfloat16,bfloat16,0,0.0716048002243042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,2,128,1,bfloat16,fp8,0,0.07237280011177064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,2,128,1,fp8,fp8,0,0.07130079865455627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,4,128,1,bfloat16,bfloat16,0,0.07345439791679383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,4,128,1,bfloat16,fp8,0,0.07528640031814575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,4,128,1,fp8,fp8,0,0.07399680018424988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,8,128,1,bfloat16,bfloat16,0,0.08316640257835388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,1,128,1,fp8,fp8,0,0.2522160053253174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,8,128,1,fp8,fp8,0,0.08658720254898071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,40,128,1,bfloat16,bfloat16,0,0.08840479850769042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,40,128,1,bfloat16,fp8,0,0.08021280169487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,1,128,1,bfloat16,bfloat16,0,0.039684799313545224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,1,128,1,bfloat16,fp8,0,0.036206400394439696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,1,128,1,fp8,fp8,0,0.03554719984531403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,40,128,1,fp8,fp8,0,0.08046240210533143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,2,128,1,bfloat16,bfloat16,0,0.03988479971885681
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,2,128,1,bfloat16,fp8,0,0.03630560040473938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,2,128,1,fp8,fp8,0,0.03627839982509613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,4,128,1,bfloat16,fp8,0,0.0383103996515274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,4,128,1,bfloat16,bfloat16,0,0.04182240068912506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,4,128,1,fp8,fp8,0,0.038099199533462524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,8,128,1,bfloat16,bfloat16,0,0.0475951999425888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,8,128,1,bfloat16,fp8,0,0.04583840072154999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,40,8,128,1,fp8,fp8,0,0.045028799772262575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,40,128,1,bfloat16,bfloat16,0,0.04880479872226715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,40,128,1,bfloat16,fp8,0,0.03949120044708252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,40,128,1,fp8,fp8,0,0.03952319920063019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,1,128,1,bfloat16,bfloat16,0,0.02208479940891266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,1,128,1,bfloat16,fp8,0,0.024139200150966645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,1,128,1,fp8,fp8,0,0.023852799832820893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,2,128,1,bfloat16,bfloat16,0,0.02221599966287613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,2,128,1,bfloat16,fp8,0,0.02372319996356964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,2,128,1,fp8,fp8,0,0.023812800645828247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,4,128,1,bfloat16,bfloat16,0,0.023024000227451324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,4,128,1,fp8,fp8,0,0.023787200450897217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,8,128,1,bfloat16,bfloat16,0,0.023078399896621703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,8,128,1,bfloat16,fp8,0,0.02414399981498718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,8,128,1,fp8,fp8,0,0.024371199309825897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,40,128,1,bfloat16,bfloat16,0,0.025179201364517213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,40,128,1,bfloat16,fp8,0,0.02409279942512512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,40,128,1,fp8,fp8,0,0.024240000545978545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,1,128,1,bfloat16,bfloat16,0,0.01746560037136078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,1,128,1,bfloat16,fp8,0,0.018374399840831758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,1,128,1,fp8,fp8,0,0.01835999935865402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,2,128,1,bfloat16,bfloat16,0,0.01749120056629181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,2,128,1,bfloat16,fp8,0,0.018572799861431122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,2,128,1,fp8,fp8,0,0.018324799835681915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,4,128,1,bfloat16,bfloat16,0,0.017407999932765962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,4,128,1,bfloat16,fp8,0,0.018668800592422485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,4,128,1,fp8,fp8,0,0.01865600049495697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,8,128,1,bfloat16,bfloat16,0,0.017470400035381316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,8,128,1,bfloat16,fp8,0,0.018478399515151976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,40,128,1,bfloat16,bfloat16,0,0.01791359931230545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,40,128,1,bfloat16,fp8,0,0.018328000605106354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,40,128,1,fp8,fp8,0,0.018385599553585052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,1,128,1,bfloat16,bfloat16,0,0.012868799269199371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,1,128,1,bfloat16,fp8,0,0.013315199315547943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,1,128,1,fp8,fp8,0,0.013326400518417358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,40,8,128,1,bfloat16,fp8,0,0.31596319675445556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,2,128,1,bfloat16,bfloat16,0,0.013395200669765472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,2,128,1,bfloat16,fp8,0,0.013337600231170654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,2,128,1,fp8,fp8,0,0.013523200154304504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,4,128,1,bfloat16,fp8,0,0.013643200695514678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,4,128,1,bfloat16,bfloat16,0,0.012971200048923492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,8,128,1,bfloat16,bfloat16,0,0.013126400113105775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,8,128,1,bfloat16,fp8,0,0.013580800592899322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,8,128,1,fp8,fp8,0,0.013894400000572205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,40,128,1,bfloat16,bfloat16,0,0.013260799646377563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,40,128,1,bfloat16,fp8,0,0.014057600498199463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,40,128,1,fp8,fp8,0,0.01390880048274994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,1,128,1,bfloat16,bfloat16,0,0.012681600451469422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,1,128,1,bfloat16,fp8,0,0.013206399977207184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,1,128,1,fp8,fp8,0,0.013451200723648072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,2,128,1,bfloat16,bfloat16,0,0.012355200201272964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,2,128,1,bfloat16,fp8,0,0.01340160071849823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,2,128,1,fp8,fp8,0,0.013025599718093871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,4,128,1,bfloat16,bfloat16,0,0.012771199643611907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,4,128,1,bfloat16,fp8,0,0.013177600502967835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,4,128,1,fp8,fp8,0,0.013291199505329133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,8,128,1,bfloat16,bfloat16,0,0.012673600018024445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,8,128,1,bfloat16,fp8,0,0.013435199856758118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,40,8,128,1,fp8,fp8,0,0.013407999277114868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,40,128,1,fp8,fp8,0,0.15256479978561402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,40,128,1,bfloat16,bfloat16,0,0.013121600449085235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,40,128,1,bfloat16,fp8,0,0.013603200018405915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,40,128,1,fp8,fp8,0,0.013609600067138673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,1,128,1,bfloat16,bfloat16,0,0.01260959953069687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,1,128,1,bfloat16,fp8,0,0.012910400331020356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,1,128,1,fp8,fp8,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,2,128,1,bfloat16,fp8,0,0.012886400520801543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,2,128,1,fp8,fp8,0,0.012747199833393097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,4,128,1,bfloat16,bfloat16,0,0.012417600303888322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,4,128,1,bfloat16,fp8,0,0.012985600531101227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,40,8,128,1,bfloat16,fp8,0,0.08570079803466797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,8,128,1,bfloat16,bfloat16,0,0.012484800070524216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,8,128,1,bfloat16,fp8,0,0.013198399543762207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,8,128,1,fp8,fp8,0,0.013099199533462525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,1,128,1,bfloat16,bfloat16,0,0.12412159442901612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,1,128,1,bfloat16,fp8,0,0.13423999547958373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,1,128,1,fp8,fp8,0,0.13249119520187377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,2,128,1,bfloat16,bfloat16,0,0.12474080324172973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,2,128,1,bfloat16,fp8,0,0.1358191967010498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,2,128,1,fp8,fp8,0,0.1357632040977478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,40,4,128,1,bfloat16,fp8,0,0.024135999381542206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,4,128,1,bfloat16,bfloat16,0,0.1455423951148987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,4,128,1,bfloat16,fp8,0,0.1518336057662964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,4,128,1,fp8,fp8,0,0.1526960015296936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,8,128,1,bfloat16,bfloat16,0,0.17851680517196655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,8,128,1,bfloat16,fp8,0,0.18820639848709106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,40,8,128,1,fp8,fp8,0,0.1888368010520935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,40,8,128,1,fp8,fp8,0,0.018771199882030486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,1,128,1,bfloat16,bfloat16,0,0.06944479942321777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,40,128,1,bfloat16,fp8,0,0.23549280166625977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,40,128,1,fp8,fp8,0,0.23773601055145263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,1,128,1,bfloat16,fp8,0,0.07241119742393494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,1,128,1,fp8,fp8,0,0.0713599979877472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,2,128,1,bfloat16,bfloat16,0,0.07023040056228638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,2,128,1,bfloat16,fp8,0,0.07310720086097718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,2,128,1,fp8,fp8,0,0.07344480156898499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,4,128,1,bfloat16,bfloat16,0,0.07954720258712769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,4,128,1,bfloat16,fp8,0,0.08258399963378907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,4,128,1,fp8,fp8,0,0.08207520246505737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,8,128,1,bfloat16,bfloat16,0,0.09601119756698609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,8,128,1,bfloat16,fp8,0,0.10181599855422974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,8,128,1,fp8,fp8,0,0.10152000188827515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,40,128,1,bfloat16,bfloat16,0,0.12344319820404052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,40,128,1,bfloat16,fp8,0,0.12544480562210084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,1,128,1,bfloat16,bfloat16,0,0.03940800130367279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,40,128,1,fp8,fp8,0,0.12449120283126831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,1,128,1,bfloat16,fp8,0,0.03654240071773529
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,1,128,1,fp8,fp8,0,0.036575999855995175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,2,128,1,bfloat16,bfloat16,0,0.038689601421356204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,2,128,1,bfloat16,fp8,0,0.03704000115394592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,2,128,1,fp8,fp8,0,0.03721440136432648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,4,128,1,bfloat16,bfloat16,0,0.04550879895687103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,4,128,1,bfloat16,fp8,0,0.04244639873504639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,4,128,1,fp8,fp8,0,0.04290240108966827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,2,128,1,bfloat16,bfloat16,0,0.012638400495052337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,8,128,1,bfloat16,bfloat16,0,0.055427199602127074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,8,128,1,bfloat16,fp8,0,0.054852801561355594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,40,8,128,1,fp8,fp8,0,0.05490080118179321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,40,128,1,bfloat16,bfloat16,0,0.0667519986629486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,40,128,1,bfloat16,fp8,0,0.06291999816894531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,40,4,128,1,fp8,fp8,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,40,128,1,fp8,fp8,0,0.06346240043640136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,1,128,1,bfloat16,bfloat16,0,0.022516800463199614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,1,128,1,bfloat16,fp8,0,0.024459199607372285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,1,128,1,fp8,fp8,0,0.024036799371242524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,2,128,1,bfloat16,bfloat16,0,0.022499200701713563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,2,128,1,fp8,fp8,0,0.024212799966335297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,4,128,1,bfloat16,bfloat16,0,0.02258719950914383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,4,128,1,bfloat16,fp8,0,0.023919999599456787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,4,128,1,fp8,fp8,0,0.024116800725460054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,8,128,1,bfloat16,bfloat16,0,0.02693760097026825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,8,128,1,bfloat16,fp8,0,0.029116800427436827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,8,128,1,fp8,fp8,0,0.028681600093841554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,40,128,1,bfloat16,bfloat16,0,0.032924801111221313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,40,128,1,bfloat16,fp8,0,0.0331712007522583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,40,128,1,fp8,fp8,0,0.033262398838996884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,1,128,1,bfloat16,bfloat16,0,0.017092800140380858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,1,128,1,bfloat16,fp8,0,0.018334400653839112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,1,128,1,fp8,fp8,0,0.018401600420475006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,2,128,1,bfloat16,bfloat16,0,0.01743520051240921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,2,128,1,bfloat16,fp8,0,0.018243199586868285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,2,128,1,fp8,fp8,0,0.018433600664138794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,4,128,1,bfloat16,bfloat16,0,0.017369599640369417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,4,128,1,bfloat16,fp8,0,0.018212799727916718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,4,128,1,fp8,fp8,0,0.01839679926633835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,8,128,1,bfloat16,bfloat16,0,0.01759999990463257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,8,128,1,bfloat16,fp8,0,0.018272000551223754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,40,8,128,1,fp8,fp8,0,0.018755200505256652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,40,128,1,bfloat16,bfloat16,0,0.021862399578094483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,40,128,1,bfloat16,fp8,0,0.02279199957847595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,40,128,1,fp8,fp8,0,0.023112000524997713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,1,128,1,bfloat16,bfloat16,0,0.013145600259304047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,1,128,1,bfloat16,fp8,0,0.013598400354385375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,1,128,1,fp8,fp8,0,0.013635200262069703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,2,128,1,bfloat16,bfloat16,0,0.013064000010490417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,2,128,1,bfloat16,fp8,0,0.0133200004696846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,2,128,1,fp8,fp8,0,0.01358720064163208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,4,128,1,bfloat16,bfloat16,0,0.013251200318336487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,4,128,1,bfloat16,fp8,0,0.013391999900341034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,4,128,1,fp8,fp8,0,0.013575999438762665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,8,128,1,bfloat16,bfloat16,0,0.013118399679660797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,8,128,1,bfloat16,fp8,0,0.013358399271965027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,40,8,128,1,fp8,fp8,0,0.013627199828624726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,40,128,1,bfloat16,bfloat16,0,0.017067199945449828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,40,128,1,bfloat16,fp8,0,0.017870399355888366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,40,128,1,fp8,fp8,0,0.017918400466442108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,1,128,1,bfloat16,bfloat16,0,0.012577599287033081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,1,128,1,bfloat16,fp8,0,0.012984000146389008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,1,128,1,fp8,fp8,0,0.013124799728393555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,40,40,128,1,bfloat16,bfloat16,0,0.23203039169311523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,2,128,1,bfloat16,fp8,0,0.012984000146389008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,2,128,1,fp8,fp8,0,0.013204799592494964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,4,128,1,bfloat16,bfloat16,0,0.012670400738716125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,4,128,1,fp8,fp8,0,0.013033600151538849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,8,128,1,bfloat16,bfloat16,0,0.012510399520397186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,8,128,1,bfloat16,fp8,0,0.013483199477195739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,8,128,1,fp8,fp8,0,0.01305440068244934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,40,128,1,bfloat16,bfloat16,0,0.012876799702644348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,40,128,1,bfloat16,fp8,0,0.013875199854373932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,40,128,1,fp8,fp8,0,0.013420799374580383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,1,128,1,bfloat16,bfloat16,0,0.012382400035858155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,1,128,1,bfloat16,fp8,0,0.013091200590133667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,1,128,1,fp8,fp8,0,0.012716799974441528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,2,128,1,bfloat16,bfloat16,0,0.012303999811410903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,2,128,1,bfloat16,fp8,0,0.013147200644016265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,2,128,1,fp8,fp8,0,0.012848000228404998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,4,128,1,bfloat16,bfloat16,0,0.01231520026922226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,4,128,1,bfloat16,fp8,0,0.01316159963607788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,4,128,1,fp8,fp8,0,0.012859199941158295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,8,128,1,bfloat16,bfloat16,0,0.012481600046157837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,8,128,1,bfloat16,fp8,0,0.013179199397563934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,40,8,128,1,fp8,fp8,0,0.012971200048923492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,40,128,1,bfloat16,bfloat16,0,0.012967999279499053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,40,128,1,bfloat16,fp8,0,0.013660800457000733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,40,128,1,fp8,fp8,0,0.013409599661827087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,1,128,1,bfloat16,bfloat16,0,0.012039999663829803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,1,128,1,bfloat16,fp8,0,0.012798400223255157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,1,128,1,fp8,fp8,0,0.012615999579429627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,2,128,1,bfloat16,bfloat16,0,0.012019199877977371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,2,128,1,bfloat16,fp8,0,0.012854400277137756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,2,128,1,fp8,fp8,0,0.012601600587368011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,4,128,1,bfloat16,bfloat16,0,0.012051200121641159
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,4,128,1,bfloat16,fp8,0,0.012980799376964568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,4,128,1,fp8,fp8,0,0.012588800489902496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,8,128,1,bfloat16,bfloat16,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,8,128,1,bfloat16,fp8,0,0.01316159963607788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,40,8,128,1,fp8,fp8,0,0.012630400061607362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,1,128,1,bfloat16,bfloat16,0,0.0716159999370575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,1,128,1,fp8,fp8,0,0.07315840125083924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,2,128,1,bfloat16,bfloat16,0,0.0778335988521576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,2,128,1,bfloat16,fp8,0,0.08209919929504395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,2,128,1,fp8,fp8,0,0.08269439935684204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,4,128,1,bfloat16,bfloat16,0,0.09079520106315613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,40,2,128,1,bfloat16,fp8,0,0.024153600633144378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,4,128,1,bfloat16,fp8,0,0.10041439533233643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,4,128,1,fp8,fp8,0,0.09973440170288086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,8,128,1,bfloat16,bfloat16,0,0.10031360387802124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,8,128,1,bfloat16,fp8,0,0.10558559894561767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,8,128,1,fp8,fp8,0,0.10478399991989136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,40,128,1,bfloat16,bfloat16,0,0.19864000082015992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,40,128,1,bfloat16,fp8,0,0.21607680320739747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,40,128,1,fp8,fp8,0,0.21356639862060547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,1,128,1,bfloat16,bfloat16,0,0.04103519916534424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,1,128,1,bfloat16,fp8,0,0.036924800276756285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,1,128,1,fp8,fp8,0,0.03682399988174438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,2,128,1,bfloat16,bfloat16,0,0.0450111985206604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,2,128,1,bfloat16,fp8,0,0.04162240028381348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,2,128,1,fp8,fp8,0,0.04149119853973389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,4,128,1,bfloat16,bfloat16,0,0.053947198390960696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,4,128,1,bfloat16,fp8,0,0.0523360013961792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,4,128,1,fp8,fp8,0,0.051836800575256345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,8,128,1,bfloat16,bfloat16,0,0.05832160115242004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,8,128,1,bfloat16,fp8,0,0.05568000078201294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,40,8,128,1,fp8,fp8,0,0.055212801694869994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,40,128,1,bfloat16,bfloat16,0,0.10653920173645019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,40,128,1,bfloat16,fp8,0,0.11106560230255128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,40,128,1,fp8,fp8,0,0.10848640203475952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,1,128,1,bfloat16,bfloat16,0,0.02282720059156418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,1,128,1,bfloat16,fp8,0,0.024558399617671967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,2,128,1,bfloat16,bfloat16,0,0.01263359934091568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,1,128,1,fp8,fp8,0,0.024247999489307403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,2,128,1,bfloat16,bfloat16,0,0.02237440049648285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,2,128,1,bfloat16,fp8,0,0.024639999866485594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,2,128,1,fp8,fp8,0,0.02407039999961853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,40,4,128,1,fp8,fp8,0,0.01342879980802536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,4,128,1,bfloat16,fp8,0,0.02921760082244873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,4,128,1,fp8,fp8,0,0.028591999411582948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,8,128,1,bfloat16,bfloat16,0,0.027076798677444457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,8,128,1,bfloat16,fp8,0,0.029227200150489806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,8,128,1,fp8,fp8,0,0.02863360047340393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,40,128,1,bfloat16,bfloat16,0,0.05188639760017395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,40,128,1,bfloat16,fp8,0,0.05642399787902832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,40,128,1,fp8,fp8,0,0.0554144024848938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,1,128,1,bfloat16,bfloat16,0,0.017761600017547608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,1,128,1,bfloat16,fp8,0,0.01897599995136261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,1,128,1,fp8,fp8,0,0.018596799671649934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,2,128,1,bfloat16,bfloat16,0,0.017560000717639922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,2,128,1,bfloat16,fp8,0,0.019012799859046935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,2,128,1,fp8,fp8,0,0.018697600066661834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,4,128,1,bfloat16,bfloat16,0,0.017739200592041017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,4,128,1,bfloat16,fp8,0,0.019041599333286287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,4,128,1,fp8,fp8,0,0.01852799952030182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,8,128,1,bfloat16,bfloat16,0,0.017862400412559508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,8,128,1,bfloat16,fp8,0,0.018779200315475465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,40,8,128,1,fp8,fp8,0,0.018243199586868285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,40,128,1,bfloat16,bfloat16,0,0.029710400104522704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,40,128,1,bfloat16,fp8,0,0.03246879875659943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,1,128,1,bfloat16,bfloat16,0,0.013097600638866424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,40,128,1,fp8,fp8,0,0.03171359896659851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,1,128,1,bfloat16,fp8,0,0.013673600554466248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,1,128,1,fp8,fp8,0,0.01363680064678192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,2,128,1,bfloat16,bfloat16,0,0.012532800436019897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,2,128,1,bfloat16,fp8,0,0.013921600580215455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,2,128,1,fp8,fp8,0,0.013655999302864074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,4,128,1,bfloat16,bfloat16,0,0.012583999335765839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,4,128,1,bfloat16,fp8,0,0.013601599633693695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,40,1,128,1,bfloat16,fp8,0,0.07423200011253357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,4,128,1,fp8,fp8,0,0.01363680064678192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,8,128,1,bfloat16,bfloat16,0,0.012417600303888322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,8,128,1,fp8,fp8,0,0.013817599415779114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,40,128,1,bfloat16,bfloat16,0,0.02101919949054718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,40,128,1,bfloat16,fp8,0,0.022441600263118745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,40,128,1,fp8,fp8,0,0.02295839935541153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,1,128,1,bfloat16,fp8,0,0.013070400059223174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,1,128,1,fp8,fp8,0,0.013288000226020813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,2,128,1,bfloat16,bfloat16,0,0.012441600114107132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,2,128,1,bfloat16,fp8,0,0.013281600177288055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,2,128,1,fp8,fp8,0,0.013198399543762207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,4,128,1,bfloat16,bfloat16,0,0.012883199751377106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,4,128,1,bfloat16,fp8,0,0.013193599879741669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,4,128,1,fp8,fp8,0,0.013491199910640716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,8,128,1,bfloat16,bfloat16,0,0.012297599762678146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,8,128,1,bfloat16,fp8,0,0.013230399787425995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,8,128,1,fp8,fp8,0,0.01279360055923462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,40,128,1,bfloat16,bfloat16,0,0.01679359972476959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,40,128,1,bfloat16,fp8,0,0.017579199373722078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,40,128,1,fp8,fp8,0,0.01812320053577423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,1,128,1,bfloat16,bfloat16,0,0.012172800302505494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,1,128,1,bfloat16,fp8,0,0.013155199587345123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,1,128,1,fp8,fp8,0,0.012851199507713318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,2,128,1,bfloat16,bfloat16,0,0.012654399871826172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,2,128,1,bfloat16,fp8,0,0.012987199425697326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,2,128,1,fp8,fp8,0,0.013262400031089782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,4,128,1,bfloat16,bfloat16,0,0.012185599654912949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,4,128,1,bfloat16,fp8,0,0.013191999495029449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,4,128,1,fp8,fp8,0,0.012862400710582733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,8,128,1,bfloat16,bfloat16,0,0.01210239976644516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,8,128,1,bfloat16,fp8,0,0.012726399302482604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,40,8,128,1,fp8,fp8,0,0.01310880035161972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,40,128,1,bfloat16,bfloat16,0,0.012788799405097962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,40,128,1,bfloat16,fp8,0,0.013897599279880523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,40,128,1,fp8,fp8,0,0.013598400354385375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,1,128,1,bfloat16,bfloat16,0,0.012203200161457062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,1,128,1,bfloat16,fp8,0,0.012643200159072877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,1,128,1,fp8,fp8,0,0.012822400033473968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,2,128,1,bfloat16,bfloat16,0,0.011955200135707856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,2,128,1,bfloat16,fp8,0,0.01297599971294403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,2,128,1,fp8,fp8,0,0.01268479973077774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,4,128,1,bfloat16,bfloat16,0,0.012319999933242797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,4,128,1,bfloat16,fp8,0,0.012891200184822083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,4,128,1,fp8,fp8,0,0.012998400628566742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,8,128,1,bfloat16,bfloat16,0,0.011744000017642975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,40,4,128,1,bfloat16,fp8,0,0.013335999846458436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,8,128,1,bfloat16,fp8,0,0.012800000607967377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,40,8,128,1,fp8,fp8,0,0.012756800651550293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,40,128,1,bfloat16,fp8,0,0.013884800672531127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,40,128,1,bfloat16,bfloat16,0,0.012995199859142303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,40,128,1,fp8,fp8,0,0.013527999818325042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,1,128,1,bfloat16,bfloat16,0,0.012044800072908401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,1,128,1,bfloat16,fp8,0,0.012782399356365205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,1,128,1,fp8,fp8,0,0.012556800246238708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,2,128,1,bfloat16,bfloat16,0,0.012003199756145477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,2,128,1,bfloat16,fp8,0,0.012865599989891053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,2,128,1,fp8,fp8,0,0.012537600100040435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,4,128,1,bfloat16,bfloat16,0,0.011972799897193909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,4,128,1,bfloat16,fp8,0,0.0127920001745224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,4,128,1,fp8,fp8,0,0.01250080019235611
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,8,128,1,bfloat16,bfloat16,0,0.011876799911260606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,8,128,1,bfloat16,fp8,0,0.012668800354003907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,40,8,128,1,fp8,fp8,0,0.012433599680662155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,1,128,1,bfloat16,bfloat16,0,0.02099040001630783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,1,128,1,bfloat16,fp8,0,0.022510400414466857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,1,128,1,fp8,fp8,0,0.022492800652980805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,2,128,1,bfloat16,bfloat16,0,0.02807680070400238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,2,128,1,bfloat16,fp8,0,0.031323200464248656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,2,128,1,fp8,fp8,0,0.031083199381828307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,4,128,1,bfloat16,fp8,0,0.04911519885063172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,4,128,1,fp8,fp8,0,0.04868960082530975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,8,128,1,bfloat16,bfloat16,0,0.07089599967002869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,8,128,1,bfloat16,fp8,0,0.08361120223999023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,8,128,1,fp8,fp8,0,0.08424800038337707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,40,8,128,1,bfloat16,fp8,0,0.013654400408267976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,40,128,1,bfloat16,bfloat16,0,0.15286719799041748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,40,1,128,1,bfloat16,bfloat16,0,0.012868799269199371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,1,128,1,bfloat16,bfloat16,0,0.014553600549697876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,1,128,1,bfloat16,fp8,0,0.01589599996805191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,1,128,1,fp8,fp8,0,0.01541920006275177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,2,128,1,bfloat16,bfloat16,0,0.01865279972553253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,2,128,1,bfloat16,fp8,0,0.019760000705718993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,2,128,1,fp8,fp8,0,0.020289599895477295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,4,128,1,bfloat16,bfloat16,0,0.02569440007209778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,4,128,1,bfloat16,fp8,0,0.02916960120201111
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,4,128,1,fp8,fp8,0,0.028723201155662535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,8,128,1,bfloat16,bfloat16,0,0.04072319865226746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,8,128,1,bfloat16,fp8,0,0.04628959894180298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,8,128,1,fp8,fp8,0,0.046795201301574704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,40,128,1,bfloat16,bfloat16,0,0.08338239789009094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,40,128,1,bfloat16,fp8,0,0.1011423945426941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,1,128,1,bfloat16,bfloat16,0,0.013329599797725678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,1,128,1,bfloat16,fp8,0,0.014183999598026275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,1,128,1,fp8,fp8,0,0.014280000329017639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,2,128,1,bfloat16,bfloat16,0,0.013484799861907959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,2,128,1,bfloat16,fp8,0,0.014211200177669525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,2,128,1,fp8,fp8,0,0.014052799344062806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,4,128,1,bfloat16,bfloat16,0,0.017020800709724428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,4,128,1,bfloat16,fp8,0,0.018592000007629395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,4,128,1,fp8,fp8,0,0.018566399812698364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,8,128,1,bfloat16,bfloat16,0,0.024307200312614442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,8,128,1,bfloat16,fp8,0,0.02752000093460083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,40,4,128,1,bfloat16,bfloat16,0,0.026344001293182373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,8,128,1,fp8,fp8,0,0.02723200023174286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,40,128,1,bfloat16,bfloat16,0,0.047542399168014525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,40,128,1,fp8,fp8,0,0.054731202125549314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,1,128,1,bfloat16,bfloat16,0,0.012800000607967377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,40,128,1,bfloat16,fp8,0,0.05507040023803711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,1,128,1,bfloat16,fp8,0,0.013102400302886962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,2,128,1,bfloat16,bfloat16,0,0.012727999687194824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,2,128,1,bfloat16,fp8,0,0.013289600610733032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,2,128,1,fp8,fp8,0,0.013499200344085693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,4,128,1,bfloat16,bfloat16,0,0.012747199833393097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,4,128,1,bfloat16,fp8,0,0.013551999628543854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,4,128,1,fp8,fp8,0,0.013606399297714233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,8,128,1,bfloat16,bfloat16,0,0.01658879965543747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,8,128,1,bfloat16,fp8,0,0.017899200320243835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,8,128,1,fp8,fp8,0,0.017929600179195405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,40,128,1,bfloat16,bfloat16,0,0.027932798862457274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,40,128,1,bfloat16,fp8,0,0.031825599074363706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,40,128,1,fp8,fp8,0,0.03172479867935181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,1,128,1,bfloat16,bfloat16,0,0.011844799667596818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,40,4,128,1,bfloat16,bfloat16,0,0.04333600103855133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,1,128,1,fp8,fp8,0,0.012641599774360657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,2,128,1,bfloat16,fp8,0,0.012771199643611907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,2,128,1,bfloat16,bfloat16,0,0.012142399698495865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,2,128,1,fp8,fp8,0,0.012707200646400452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,4,128,1,bfloat16,bfloat16,0,0.01226079985499382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,4,128,1,bfloat16,fp8,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,4,128,1,fp8,fp8,0,0.012984000146389008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,8,128,1,bfloat16,bfloat16,0,0.012081599980592727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,8,128,1,bfloat16,fp8,0,0.013195200264453888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,8,128,1,fp8,fp8,0,0.013041600584983826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,40,128,1,bfloat16,bfloat16,0,0.02060000002384186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,40,128,1,bfloat16,fp8,0,0.18816640377044677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,40,128,1,bfloat16,fp8,0,0.022526399791240694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,40,128,1,fp8,fp8,0,0.022606399655342103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,40,40,128,1,fp8,fp8,0,0.18823200464248657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,1,128,1,bfloat16,fp8,0,0.0125231996178627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,1,128,1,fp8,fp8,0,0.012580800056457519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,2,128,1,bfloat16,bfloat16,0,0.012003199756145477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,2,128,1,bfloat16,fp8,0,0.012561599910259246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,2,128,1,fp8,fp8,0,0.012651200592517852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,4,128,1,bfloat16,bfloat16,0,0.012176000326871873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,4,128,1,bfloat16,fp8,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,4,128,1,fp8,fp8,0,0.012800000607967377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,8,128,1,bfloat16,bfloat16,0,0.012198399752378464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,8,128,1,bfloat16,fp8,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,40,128,1,bfloat16,bfloat16,0,0.016542400419712066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,40,128,1,bfloat16,fp8,0,0.017679999768733978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,40,128,1,fp8,fp8,0,0.01780959963798523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,1,128,1,bfloat16,bfloat16,0,0.011524800211191177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,1,128,1,bfloat16,fp8,0,0.012414400279521943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,1,128,1,fp8,fp8,0,0.012302400171756744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,2,128,1,bfloat16,bfloat16,0,0.01178399994969368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,2,128,1,bfloat16,fp8,0,0.012382400035858155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,2,128,1,fp8,fp8,0,0.012412799894809723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,4,128,1,bfloat16,bfloat16,0,0.011718399822711945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,4,128,1,bfloat16,fp8,0,0.01250240057706833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,4,128,1,fp8,fp8,0,0.012436799705028534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,8,128,1,bfloat16,bfloat16,0,0.012033600360155106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,8,128,1,bfloat16,fp8,0,0.01268479973077774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,40,8,128,1,fp8,fp8,0,0.012928000092506409
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,40,128,1,bfloat16,bfloat16,0,0.01292639970779419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,40,128,1,bfloat16,fp8,0,0.013769599795341491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,40,128,1,fp8,fp8,0,0.013767999410629273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,1,128,1,bfloat16,bfloat16,0,0.011612799763679505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,1,128,1,bfloat16,fp8,0,0.01225920021533966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,1,128,1,fp8,fp8,0,0.012273599952459335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,2,128,1,bfloat16,bfloat16,0,0.011617600172758102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,2,128,1,bfloat16,fp8,0,0.012350399792194367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,2,128,1,fp8,fp8,0,0.012457600235939026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,4,128,1,bfloat16,bfloat16,0,0.011707200109958649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,40,1,128,1,fp8,fp8,0,0.013579200208187103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,4,128,1,fp8,fp8,0,0.012328000366687774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,8,128,1,bfloat16,bfloat16,0,0.01170239970088005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,8,128,1,bfloat16,fp8,0,0.01228799968957901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,8,128,1,fp8,fp8,0,0.01231200024485588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,40,128,1,bfloat16,bfloat16,0,0.012556800246238708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,40,128,1,bfloat16,fp8,0,0.013324800133705138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,40,128,1,fp8,fp8,0,0.013352000713348388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,40,1,128,1,bfloat16,fp8,0,0.01252640038728714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,1,128,1,bfloat16,bfloat16,0,0.011419200152158738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,1,128,1,bfloat16,fp8,0,0.012216000258922577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,1,128,1,fp8,fp8,0,0.012148799747228623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,2,128,1,bfloat16,bfloat16,0,0.01141280010342598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,2,128,1,bfloat16,fp8,0,0.012116800248622894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,2,128,1,fp8,fp8,0,0.012379200011491776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,4,128,1,bfloat16,bfloat16,0,0.011420799791812897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,4,128,1,bfloat16,fp8,0,0.01215839982032776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,4,128,1,fp8,fp8,0,0.012336000055074691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,8,128,1,bfloat16,bfloat16,0,0.011502400040626526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,8,128,1,bfloat16,fp8,0,0.012169600278139115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,40,8,128,1,fp8,fp8,0,0.012366399914026261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,1,128,1,bfloat16,bfloat16,0,0.011873599886894227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,40,8,128,1,fp8,fp8,0,0.012827199697494508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,40,40,128,1,fp8,fp8,0,0.1
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,40,4,128,1,bfloat16,fp8,0,0.01250240057706833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,1,128,1,fp8,fp8,0,24.746035766601562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,1,128,1,bfloat16,fp8,0,25.146131896972655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,2,128,1,bfloat16,fp8,0,24.791075134277342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,2,128,1,fp8,fp8,0,24.954588317871092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,1,128,1,bfloat16,bfloat16,0,28.88929138183594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,2,128,1,bfloat16,bfloat16,0,29.10057678222656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,4,128,1,bfloat16,fp8,0,24.70063171386719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,4,128,1,bfloat16,bfloat16,0,29.594485473632812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,32,128,1,bfloat16,bfloat16,0,17.187557983398438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,1,128,1,bfloat16,bfloat16,0,16.32476348876953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,4,128,1,fp8,fp8,0,27.109515380859374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,8,128,1,bfloat16,fp8,0,26.891766357421876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,8,128,1,fp8,fp8,0,27.189825439453124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,32,8,128,1,bfloat16,bfloat16,0,32.11585388183594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,1,128,1,bfloat16,fp8,0,12.261032104492188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,1,128,1,fp8,fp8,0,12.343694305419922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,2,128,1,bfloat16,fp8,0,12.365570831298829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,2,128,1,bfloat16,bfloat16,0,14.775373840332032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,2,128,1,fp8,fp8,0,13.094549560546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,4,128,1,bfloat16,fp8,0,12.394744110107421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,4,128,1,bfloat16,bfloat16,0,15.680329895019531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,4,128,1,fp8,fp8,0,12.751493072509765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,32,128,1,bfloat16,bfloat16,0,8.675003051757812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,8,128,1,fp8,fp8,0,13.193963623046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,8,128,1,bfloat16,fp8,0,13.496864318847656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,8,128,1,bfloat16,bfloat16,0,16.227859497070312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,1,128,1,bfloat16,fp8,0,6.574877166748047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,1,128,1,fp8,fp8,0,6.460205078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,1,128,1,bfloat16,bfloat16,0,7.677188873291016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,2,128,1,bfloat16,bfloat16,0,7.463483428955078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,2,128,1,fp8,fp8,0,6.34300651550293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,2,128,1,bfloat16,fp8,0,6.613275146484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,4,128,1,bfloat16,bfloat16,0,7.399822235107422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,4,128,1,bfloat16,fp8,0,6.252632141113281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,4,128,1,fp8,fp8,0,6.2719776153564455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,8,128,1,bfloat16,bfloat16,0,7.866423797607422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,8,128,1,bfloat16,fp8,0,6.232136154174805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,8,128,1,fp8,fp8,0,6.225491333007812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,32,128,1,bfloat16,bfloat16,0,3.868264007568359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,1,128,1,bfloat16,bfloat16,0,3.6956607818603517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,1,128,1,bfloat16,fp8,0,3.3537521362304688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,1,128,1,fp8,fp8,0,3.6789775848388673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,2,128,1,bfloat16,bfloat16,0,3.8333263397216797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,2,128,1,bfloat16,fp8,0,3.2058479309082033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,2,128,1,fp8,fp8,0,3.3730640411376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,4,128,1,bfloat16,bfloat16,0,3.668889617919922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,4,128,1,bfloat16,fp8,0,3.24627685546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,4,128,1,fp8,fp8,0,3.4940975189208983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,8,128,1,bfloat16,bfloat16,0,3.842700958251953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,8,128,1,bfloat16,fp8,0,3.1422096252441407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,8,128,1,fp8,fp8,0,3.3405841827392577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,1,128,1,bfloat16,fp8,0,14.481378173828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,1,128,1,bfloat16,bfloat16,0,17.373568725585937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,1,128,1,fp8,fp8,0,14.962408447265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,2,128,1,bfloat16,bfloat16,0,17.74993896484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,2,128,1,bfloat16,fp8,0,14.757847595214844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,2,128,1,fp8,fp8,0,14.359884643554688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,4,128,1,bfloat16,bfloat16,0,17.632762145996093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,4,128,1,bfloat16,fp8,0,14.635543823242188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,4,128,1,fp8,fp8,0,14.76934814453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,8,128,1,bfloat16,bfloat16,0,17.985272216796876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,32,128,1,bfloat16,fp8,0,3.226567840576172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,32,32,128,1,fp8,fp8,0,3.195534324645996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,32,128,1,bfloat16,fp8,0,6.418468475341797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,32,32,128,1,fp8,fp8,0,6.407449340820312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,8,128,1,bfloat16,fp8,0,14.70516357421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,32,128,1,bfloat16,fp8,0,12.734528350830079
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,32,32,128,1,fp8,fp8,0,12.733080291748047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,32,128,1,bfloat16,fp8,0,7.289070129394531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,32,8,128,1,fp8,fp8,0,14.324972534179688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,32,128,1,bfloat16,bfloat16,0,9.126886749267578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,32,128,1,fp8,fp8,0,7.301139068603516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,1,128,1,bfloat16,fp8,0,7.02929458618164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,1,128,1,fp8,fp8,0,7.0082847595214846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,1,128,1,bfloat16,bfloat16,0,9.086856079101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,2,128,1,bfloat16,bfloat16,0,8.39902572631836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,2,128,1,bfloat16,fp8,0,7.268785858154297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,2,128,1,fp8,fp8,0,7.18564453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,4,128,1,bfloat16,fp8,0,7.043527984619141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,4,128,1,bfloat16,bfloat16,0,8.419554901123046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,4,128,1,fp8,fp8,0,7.049716949462891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,32,128,1,bfloat16,fp8,0,3.7264591217041017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,32,128,1,fp8,fp8,0,3.662015914916992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,32,128,1,bfloat16,bfloat16,0,4.703494262695313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,1,128,1,bfloat16,bfloat16,0,4.151462554931641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,8,128,1,bfloat16,fp8,0,7.4179023742675785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,8,128,1,bfloat16,bfloat16,0,8.749073791503907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,32,8,128,1,fp8,fp8,0,7.095954895019531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,1,128,1,fp8,fp8,0,3.5446895599365233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,2,128,1,bfloat16,bfloat16,0,4.100246429443359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,2,128,1,bfloat16,fp8,0,4.05909423828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,2,128,1,fp8,fp8,0,3.6448463439941405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,4,128,1,bfloat16,fp8,0,3.7115505218505858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,4,128,1,bfloat16,bfloat16,0,4.5485889434814455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,4,128,1,fp8,fp8,0,3.522518539428711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,1,128,1,bfloat16,fp8,0,3.5614273071289064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,32,128,1,bfloat16,bfloat16,0,2.2336847305297853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,8,128,1,bfloat16,bfloat16,0,4.460083389282227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,32,128,1,bfloat16,fp8,0,2.1261167526245117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,8,128,1,fp8,fp8,0,3.56005744934082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,32,128,1,fp8,fp8,0,1.986851119995117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,1,128,1,bfloat16,bfloat16,0,1.9023199081420898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,32,8,128,1,bfloat16,fp8,0,4.141596984863281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,1,128,1,bfloat16,fp8,0,1.7560543060302733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,1,128,1,fp8,fp8,0,2.1018447875976562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,2,128,1,bfloat16,bfloat16,0,2.1744016647338866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,2,128,1,fp8,fp8,0,1.7875680923461914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,2,128,1,bfloat16,fp8,0,2.0679664611816406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,4,128,1,bfloat16,bfloat16,0,1.9950239181518554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,4,128,1,fp8,fp8,0,2.105183982849121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,4,128,1,bfloat16,fp8,0,2.4354175567626952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,8,128,1,bfloat16,bfloat16,0,1.9965375900268554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,8,128,1,bfloat16,fp8,0,2.0367984771728516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,32,8,128,1,fp8,fp8,0,1.9419343948364258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,1,128,1,fp8,fp8,0,10.0145263671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,1,128,1,bfloat16,fp8,0,10.298616027832031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,2,128,1,fp8,fp8,0,10.127204895019531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,2,128,1,bfloat16,fp8,0,10.50549087524414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,1,128,1,bfloat16,bfloat16,0,12.075587463378906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,4,128,1,bfloat16,fp8,0,10.702521514892577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,2,128,1,bfloat16,bfloat16,0,12.293013000488282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,4,128,1,bfloat16,bfloat16,0,12.310906982421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,32,128,1,fp8,fp8,0,5.274822235107422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,32,128,1,bfloat16,bfloat16,0,6.430843353271484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,32,128,1,bfloat16,fp8,0,5.64323844909668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,1,128,1,bfloat16,bfloat16,0,5.859641647338867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,4,128,1,fp8,fp8,0,10.504564666748047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,8,128,1,bfloat16,fp8,0,9.988565063476562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,8,128,1,fp8,fp8,0,10.006537628173827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,32,8,128,1,bfloat16,bfloat16,0,12.35157470703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,1,128,1,bfloat16,fp8,0,4.927158355712891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,1,128,1,fp8,fp8,0,5.000417709350586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,2,128,1,bfloat16,bfloat16,0,6.140268707275391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,2,128,1,bfloat16,fp8,0,5.098846435546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,2,128,1,fp8,fp8,0,5.197611236572266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,4,128,1,bfloat16,fp8,0,4.962144088745117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,4,128,1,bfloat16,bfloat16,0,5.759524917602539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,4,128,1,fp8,fp8,0,4.97935676574707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,32,128,1,bfloat16,bfloat16,0,3.22684326171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,8,128,1,bfloat16,fp8,0,5.43948974609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,8,128,1,bfloat16,bfloat16,0,5.918587112426758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,32,128,1,fp8,fp8,0,2.6034591674804686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,32,8,128,1,fp8,fp8,0,5.103078460693359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,1,128,1,bfloat16,bfloat16,0,3.4694080352783203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,1,128,1,bfloat16,fp8,0,2.467558479309082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,1,128,1,fp8,fp8,0,2.907208061218262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,2,128,1,fp8,fp8,0,2.470078468322754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,2,128,1,bfloat16,fp8,0,2.7019615173339844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,2,128,1,bfloat16,bfloat16,0,2.9830415725708006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,4,128,1,bfloat16,bfloat16,0,3.099847984313965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,4,128,1,fp8,fp8,0,2.531167984008789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,4,128,1,bfloat16,fp8,0,3.146724891662598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,32,128,1,bfloat16,bfloat16,0,1.706590461730957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,32,128,1,bfloat16,fp8,0,1.6203983306884766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,8,128,1,bfloat16,fp8,0,2.5209312438964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,32,128,1,fp8,fp8,0,1.3838047981262207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,8,128,1,fp8,fp8,0,2.6488672256469727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,8,128,1,bfloat16,bfloat16,0,3.034774398803711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,1,128,1,bfloat16,bfloat16,0,1.3620672225952148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,1,128,1,bfloat16,fp8,0,1.4463168144226075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,2,128,1,bfloat16,fp8,0,1.3109999656677247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,2,128,1,bfloat16,bfloat16,0,1.3981552124023438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,2,128,1,fp8,fp8,0,1.2493680000305176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,4,128,1,bfloat16,bfloat16,0,1.580174446105957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,32,32,128,1,bfloat16,fp8,0,2.674148750305176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,4,128,1,bfloat16,fp8,0,1.4190848350524903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,4,128,1,fp8,fp8,0,1.3189200401306151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,8,128,1,fp8,fp8,0,1.3190640449523925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,8,128,1,bfloat16,bfloat16,0,1.4552528381347656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,8,128,1,bfloat16,fp8,0,1.4684639930725099
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,32,1,128,1,fp8,fp8,0,1.3373711585998536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,1,128,1,fp8,fp8,0,12.938774108886719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,1,128,1,bfloat16,fp8,0,13.251780700683593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,2,128,1,bfloat16,fp8,0,13.129147338867188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,2,128,1,fp8,fp8,0,13.337544250488282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,1,128,1,bfloat16,bfloat16,0,15.193606567382812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,2,128,1,bfloat16,bfloat16,0,16.285429382324217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,4,128,1,bfloat16,bfloat16,0,15.830644226074218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,4,128,1,bfloat16,fp8,0,13.009927368164062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,32,128,1,bfloat16,bfloat16,0,8.23760986328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,32,128,1,bfloat16,fp8,0,6.9068962097167965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,32,128,1,fp8,fp8,0,6.947207641601563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,4,128,1,fp8,fp8,0,13.513053894042969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,8,128,1,fp8,fp8,0,13.074241638183594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,8,128,1,bfloat16,fp8,0,13.40320281982422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,1,128,1,bfloat16,bfloat16,0,7.828228759765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,32,8,128,1,bfloat16,bfloat16,0,15.564590454101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,1,128,1,bfloat16,fp8,0,6.5076141357421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,1,128,1,fp8,fp8,0,6.441567993164062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,2,128,1,bfloat16,fp8,0,6.777649688720703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,2,128,1,bfloat16,bfloat16,0,8.059063720703126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,2,128,1,fp8,fp8,0,6.446862030029297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,4,128,1,bfloat16,fp8,0,6.902035522460937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,4,128,1,bfloat16,bfloat16,0,8.413428497314452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,4,128,1,fp8,fp8,0,6.858837127685547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,8,128,1,bfloat16,fp8,0,6.753358459472656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,32,128,1,bfloat16,fp8,0,3.614379119873047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,32,128,1,bfloat16,bfloat16,0,4.845920181274414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,8,128,1,bfloat16,bfloat16,0,9.020990753173828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,32,128,1,fp8,fp8,0,3.7419456481933593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,1,128,1,bfloat16,fp8,0,3.317915344238281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,32,8,128,1,fp8,fp8,0,6.733204650878906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,1,128,1,bfloat16,bfloat16,0,4.157287979125977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,1,128,1,fp8,fp8,0,3.3319728851318358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,2,128,1,bfloat16,bfloat16,0,3.9952911376953124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,2,128,1,fp8,fp8,0,3.4571601867675783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,2,128,1,bfloat16,fp8,0,3.636783981323242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,4,128,1,bfloat16,fp8,0,3.311979293823242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,4,128,1,fp8,fp8,0,3.2691009521484373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,4,128,1,bfloat16,bfloat16,0,4.809708786010742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,8,128,1,bfloat16,bfloat16,0,4.0359455108642575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,8,128,1,bfloat16,fp8,0,3.465564727783203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,32,128,1,bfloat16,fp8,0,1.7459632873535156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,32,128,1,bfloat16,bfloat16,0,1.9747983932495117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,32,128,1,fp8,fp8,0,1.7226079940795898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,1,128,1,bfloat16,bfloat16,0,1.8919664382934571
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,32,8,128,1,fp8,fp8,0,3.7471920013427735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,2,128,1,bfloat16,fp8,0,1.6805248260498047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,1,128,1,bfloat16,fp8,0,1.8969104766845704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,1,128,1,fp8,fp8,0,1.761510467529297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,2,128,1,bfloat16,bfloat16,0,1.8781984329223633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,2,128,1,fp8,fp8,0,1.7406736373901368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,4,128,1,bfloat16,bfloat16,0,1.8523696899414062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,4,128,1,bfloat16,fp8,0,1.8562480926513671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,32,128,1,bfloat16,fp8,0,0.9319600105285645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,32,128,1,bfloat16,bfloat16,0,1.105299186706543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,4,128,1,fp8,fp8,0,1.6655263900756836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,32,128,1,fp8,fp8,0,1.0023951530456543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,8,128,1,bfloat16,fp8,0,1.6689775466918946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,8,128,1,fp8,fp8,0,1.6350784301757812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,32,8,128,1,bfloat16,bfloat16,0,1.9604799270629882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,1,128,1,bfloat16,fp8,0,0.83920316696167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,1,128,1,bfloat16,bfloat16,0,0.9481663703918457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,2,128,1,bfloat16,bfloat16,0,0.8721023559570312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,1,128,1,fp8,fp8,0,0.9459296226501465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,2,128,1,bfloat16,fp8,0,0.8908271789550781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,2,128,1,fp8,fp8,0,0.8132543563842773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,4,128,1,bfloat16,bfloat16,0,0.8903519630432128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,4,128,1,bfloat16,fp8,0,0.969974422454834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,4,128,1,fp8,fp8,0,0.8370816230773925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,8,128,1,bfloat16,bfloat16,0,0.9070912361145019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,8,128,1,bfloat16,fp8,0,0.8928000450134277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,32,8,128,1,fp8,fp8,0,0.9583264350891113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,1,128,1,fp8,fp8,0,7.613404846191406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,2,128,1,bfloat16,fp8,0,7.510433959960937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,2,128,1,fp8,fp8,0,7.606916809082032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,1,128,1,bfloat16,bfloat16,0,8.73907012939453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,4,128,1,bfloat16,fp8,0,7.690948486328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,2,128,1,bfloat16,bfloat16,0,9.075121307373047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,4,128,1,bfloat16,bfloat16,0,8.985601806640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,1,128,1,bfloat16,fp8,0,7.9157554626464846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,32,128,1,bfloat16,bfloat16,0,4.930887985229492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,32,128,1,bfloat16,fp8,0,4.081745529174805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,32,128,1,fp8,fp8,0,4.014427185058594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,4,128,1,fp8,fp8,0,7.470829010009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,8,128,1,bfloat16,fp8,0,7.7749267578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,1,128,1,bfloat16,bfloat16,0,4.46495361328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,8,128,1,fp8,fp8,0,7.590624237060547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,32,8,128,1,bfloat16,bfloat16,0,9.088349151611329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,1,128,1,fp8,fp8,0,3.7521903991699217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,1,128,1,bfloat16,fp8,0,3.900774383544922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,2,128,1,bfloat16,bfloat16,0,4.326508712768555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,2,128,1,bfloat16,fp8,0,3.765691375732422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,2,128,1,fp8,fp8,0,3.883638381958008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,4,128,1,bfloat16,fp8,0,3.8365966796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,4,128,1,bfloat16,bfloat16,0,4.401174545288086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,4,128,1,fp8,fp8,0,3.890787124633789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,8,128,1,bfloat16,fp8,0,3.797524642944336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,32,128,1,bfloat16,bfloat16,0,2.5505775451660155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,8,128,1,bfloat16,bfloat16,0,4.684139251708984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,32,8,128,1,fp8,fp8,0,3.9990001678466798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,32,128,1,bfloat16,fp8,0,2.079220771789551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,32,128,1,fp8,fp8,0,2.1329824447631838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,1,128,1,bfloat16,bfloat16,0,2.2379695892333986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,1,128,1,bfloat16,fp8,0,1.8721952438354492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,2,128,1,bfloat16,fp8,0,1.8685407638549805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,2,128,1,fp8,fp8,0,1.9138944625854493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,2,128,1,bfloat16,bfloat16,0,2.5231679916381835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,4,128,1,bfloat16,bfloat16,0,2.358011245727539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,4,128,1,fp8,fp8,0,1.878121566772461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,4,128,1,bfloat16,fp8,0,2.230808067321777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,8,128,1,bfloat16,bfloat16,0,2.190328025817871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,8,128,1,bfloat16,fp8,0,1.9123039245605469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,32,128,1,bfloat16,bfloat16,0,1.3387040138244628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,32,128,1,bfloat16,fp8,0,1.0163887977600097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,32,128,1,fp8,fp8,0,1.0876655578613281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,8,128,1,fp8,fp8,0,2.189129638671875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,1,128,1,bfloat16,fp8,0,0.947539234161377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,1,128,1,fp8,fp8,0,1.0699407577514648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,1,128,1,bfloat16,bfloat16,0,1.2214207649230957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,2,128,1,bfloat16,bfloat16,0,1.0624752044677734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,2,128,1,bfloat16,fp8,0,0.9548831939697265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,32,1,128,1,fp8,fp8,0,2.2057056427001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,2,128,1,fp8,fp8,0,1.0567952156066895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,4,128,1,bfloat16,bfloat16,0,1.0328127861022949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,4,128,1,bfloat16,fp8,0,0.9953632354736328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,8,128,1,bfloat16,fp8,0,0.9534591674804688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,32,128,1,bfloat16,bfloat16,0,0.6121679782867432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,8,128,1,fp8,fp8,0,0.9528719902038574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,4,128,1,fp8,fp8,0,1.1314111709594727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,32,8,128,1,bfloat16,bfloat16,0,1.1666303634643556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,32,128,1,fp8,fp8,0,0.5832479953765869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,1,128,1,bfloat16,bfloat16,0,0.5161439895629882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,1,128,1,bfloat16,fp8,0,0.47422080039978026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,32,128,1,bfloat16,fp8,0,0.5229695796966553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,1,128,1,fp8,fp8,0,0.5115407943725586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,2,128,1,bfloat16,bfloat16,0,0.5105855941772461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,2,128,1,bfloat16,fp8,0,0.521065616607666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,2,128,1,fp8,fp8,0,0.560368013381958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,4,128,1,bfloat16,fp8,0,0.49367680549621584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,4,128,1,bfloat16,bfloat16,0,0.5988319873809814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,4,128,1,fp8,fp8,0,0.4768928050994873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,8,128,1,bfloat16,bfloat16,0,0.5277855873107911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,8,128,1,bfloat16,fp8,0,0.5037824153900147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,32,8,128,1,fp8,fp8,0,0.48155999183654785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,1,128,1,bfloat16,fp8,0,7.267388916015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,1,128,1,fp8,fp8,0,7.185076904296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,2,128,1,fp8,fp8,0,7.07713623046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,4,128,1,bfloat16,fp8,0,7.097856140136718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,2,128,1,bfloat16,fp8,0,7.3224433898925785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,1,128,1,bfloat16,bfloat16,0,8.452642822265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,2,128,1,bfloat16,bfloat16,0,8.39725570678711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,4,128,1,bfloat16,bfloat16,0,8.756970977783203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,32,128,1,bfloat16,fp8,0,3.8940990447998045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,32,128,1,bfloat16,bfloat16,0,4.841758346557617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,32,128,1,fp8,fp8,0,3.8935184478759766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,1,128,1,bfloat16,bfloat16,0,4.1377616882324215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,4,128,1,fp8,fp8,0,7.171812438964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,8,128,1,bfloat16,fp8,0,7.118052673339844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,8,128,1,fp8,fp8,0,7.10048828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,1,128,1,bfloat16,fp8,0,3.497087860107422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,32,8,128,1,bfloat16,bfloat16,0,8.944992065429688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,1,128,1,fp8,fp8,0,3.5383487701416017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,2,128,1,bfloat16,bfloat16,0,4.315840148925782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,2,128,1,bfloat16,fp8,0,3.6481582641601564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,2,128,1,fp8,fp8,0,3.583201599121094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,4,128,1,bfloat16,fp8,0,3.5236000061035155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,4,128,1,bfloat16,bfloat16,0,4.020670318603516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,32,128,1,bfloat16,bfloat16,0,2.3235536575317384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,4,128,1,fp8,fp8,0,3.5237472534179686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,8,128,1,bfloat16,fp8,0,3.567780685424805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,32,128,1,bfloat16,fp8,0,1.9678239822387695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,32,128,1,fp8,fp8,0,1.9431743621826172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,8,128,1,fp8,fp8,0,3.573236846923828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,32,8,128,1,bfloat16,bfloat16,0,4.3242431640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,1,128,1,bfloat16,bfloat16,0,2.0023632049560547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,1,128,1,fp8,fp8,0,1.7475519180297852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,1,128,1,bfloat16,fp8,0,1.9812671661376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,2,128,1,bfloat16,bfloat16,0,1.9486495971679687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,2,128,1,bfloat16,fp8,0,1.7662672042846679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,2,128,1,fp8,fp8,0,1.7493631362915039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,4,128,1,bfloat16,bfloat16,0,2.016076850891113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,4,128,1,fp8,fp8,0,2.0218896865844727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,8,128,1,bfloat16,bfloat16,0,1.9694175720214844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,32,128,1,bfloat16,fp8,0,0.9823967933654785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,8,128,1,fp8,fp8,0,1.9633520126342774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,32,128,1,fp8,fp8,0,1.1035504341125488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,1,128,1,bfloat16,bfloat16,0,1.0150079727172852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,1,128,1,fp8,fp8,0,0.8795280456542969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,1,128,1,bfloat16,fp8,0,0.9523967742919922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,2,128,1,bfloat16,fp8,0,0.9335824012756347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,2,128,1,bfloat16,bfloat16,0,0.9616496086120605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,2,128,1,fp8,fp8,0,0.9736319541931152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,4,128,1,bfloat16,bfloat16,0,0.9749232292175293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,4,128,1,bfloat16,fp8,0,0.9753456115722656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,4,128,1,fp8,fp8,0,0.9355744361877442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,8,128,1,bfloat16,fp8,0,0.9007599830627442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,8,128,1,bfloat16,bfloat16,0,0.9967743873596191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,4,128,1,bfloat16,fp8,0,1.9242128372192382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,8,128,1,fp8,fp8,0,0.9549087524414063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,32,128,1,bfloat16,fp8,0,0.4987055778503418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,32,128,1,bfloat16,bfloat16,0,0.565447998046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,32,128,1,fp8,fp8,0,0.5456975936889649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,1,128,1,bfloat16,bfloat16,0,0.47287521362304685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,32,32,128,1,bfloat16,bfloat16,0,1.133619213104248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,1,128,1,bfloat16,fp8,0,0.4749567985534668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,1,128,1,fp8,fp8,0,0.446889591217041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,2,128,1,bfloat16,bfloat16,0,0.47618880271911623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,2,128,1,bfloat16,fp8,0,0.5228623867034912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,2,128,1,fp8,fp8,0,0.44752159118652346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,4,128,1,bfloat16,fp8,0,0.4654672145843506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,4,128,1,bfloat16,bfloat16,0,0.4871551990509033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,4,128,1,fp8,fp8,0,0.4507711887359619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,8,128,1,bfloat16,bfloat16,0,0.4933487892150879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,8,128,1,bfloat16,fp8,0,0.49950399398803713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,32,128,1,bfloat16,bfloat16,0,0.28902719020843504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,32,128,1,bfloat16,fp8,0,0.2624847888946533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,1,128,1,bfloat16,bfloat16,0,0.24325120449066162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,1,128,1,bfloat16,fp8,0,0.24714078903198242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,1,128,1,fp8,fp8,0,0.2284559965133667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,2,128,1,bfloat16,bfloat16,0,0.24784319400787352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,2,128,1,bfloat16,fp8,0,0.2313760042190552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,2,128,1,fp8,fp8,0,0.2413872003555298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,4,128,1,bfloat16,bfloat16,0,0.2439215898513794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,4,128,1,fp8,fp8,0,0.23217439651489258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,8,128,1,bfloat16,bfloat16,0,0.2587631940841675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,8,128,1,bfloat16,fp8,0,0.2334575891494751
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,8,128,1,fp8,fp8,0,0.23501760959625245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,32,8,128,1,bfloat16,fp8,0,1.853348731994629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,32,8,128,1,fp8,fp8,0,0.45658721923828127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,4,128,1,bfloat16,fp8,0,0.23789920806884765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,1,128,1,bfloat16,fp8,0,4.190902328491211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,1,128,1,fp8,fp8,0,4.217873764038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,1,128,1,bfloat16,bfloat16,0,5.141124725341797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,32,32,128,1,fp8,fp8,0,0.2559439897537231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,2,128,1,bfloat16,bfloat16,0,4.858713531494141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,2,128,1,bfloat16,fp8,0,4.157187271118164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,2,128,1,fp8,fp8,0,4.250758361816406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,4,128,1,bfloat16,bfloat16,0,4.886870574951172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,4,128,1,bfloat16,fp8,0,4.204956817626953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,4,128,1,fp8,fp8,0,4.190339279174805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,32,128,1,bfloat16,bfloat16,0,2.863676834106445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,32,128,1,bfloat16,fp8,0,2.370395278930664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,8,128,1,bfloat16,bfloat16,0,5.222267150878906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,8,128,1,bfloat16,fp8,0,4.270587158203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,1,128,1,bfloat16,fp8,0,2.081475257873535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,32,128,1,fp8,fp8,0,2.4007280349731444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,1,128,1,bfloat16,bfloat16,0,2.305771255493164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,32,8,128,1,fp8,fp8,0,4.360539245605469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,1,128,1,fp8,fp8,0,2.2246992111206056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,2,128,1,bfloat16,bfloat16,0,2.361444854736328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,2,128,1,bfloat16,fp8,0,2.219100761413574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,2,128,1,fp8,fp8,0,2.086787223815918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,4,128,1,bfloat16,fp8,0,2.100409507751465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,4,128,1,bfloat16,bfloat16,0,2.3687616348266602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,4,128,1,fp8,fp8,0,2.2472208023071287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,8,128,1,bfloat16,bfloat16,0,2.467145538330078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,32,128,1,bfloat16,bfloat16,0,1.4033455848693848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,32,128,1,bfloat16,fp8,0,1.195969581604004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,32,128,1,fp8,fp8,0,1.194883155822754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,1,128,1,bfloat16,fp8,0,1.0457311630249024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,8,128,1,bfloat16,fp8,0,2.1642175674438477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,32,8,128,1,fp8,fp8,0,2.229078483581543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,1,128,1,fp8,fp8,0,1.1912015914916991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,2,128,1,bfloat16,bfloat16,0,1.1868127822875976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,2,128,1,bfloat16,fp8,0,1.0479647636413574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,2,128,1,fp8,fp8,0,1.0474512100219726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,4,128,1,bfloat16,bfloat16,0,1.1317999839782715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,4,128,1,bfloat16,fp8,0,1.0561087608337403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,4,128,1,fp8,fp8,0,1.0564000129699707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,8,128,1,bfloat16,bfloat16,0,1.2015711784362793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,32,128,1,bfloat16,bfloat16,0,0.6969535827636719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,32,128,1,fp8,fp8,0,0.6034863948822021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,1,128,1,bfloat16,bfloat16,0,0.555134391784668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,8,128,1,bfloat16,fp8,0,1.1725824356079102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,8,128,1,fp8,fp8,0,1.0712976455688477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,1,128,1,bfloat16,fp8,0,0.6050623893737793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,1,128,1,fp8,fp8,0,0.5398736000061035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,2,128,1,bfloat16,bfloat16,0,0.5618080139160156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,2,128,1,bfloat16,fp8,0,0.539683198928833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,2,128,1,fp8,fp8,0,0.5660096168518066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,4,128,1,bfloat16,bfloat16,0,0.5698480129241943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,4,128,1,bfloat16,fp8,0,0.5787216186523437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,4,128,1,fp8,fp8,0,0.5455039978027344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,8,128,1,bfloat16,bfloat16,0,0.5878111839294433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,8,128,1,bfloat16,fp8,0,0.5425504207611084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,8,128,1,fp8,fp8,0,0.5635087966918946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,32,128,1,bfloat16,fp8,0,0.32200160026550295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,32,128,1,fp8,fp8,0,0.31407999992370605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,1,128,1,bfloat16,bfloat16,0,0.2855776071548462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,1,128,1,bfloat16,fp8,0,0.28320960998535155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,1,128,1,fp8,fp8,0,0.2775039911270142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,2,128,1,bfloat16,bfloat16,0,0.2980015993118286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,2,128,1,bfloat16,fp8,0,0.2772975921630859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,2,128,1,fp8,fp8,0,0.2740367889404297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,4,128,1,bfloat16,bfloat16,0,0.2969264030456543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,4,128,1,bfloat16,fp8,0,0.2804464101791382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,4,128,1,fp8,fp8,0,0.28010399341583253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,8,128,1,bfloat16,bfloat16,0,0.2994335889816284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,8,128,1,bfloat16,fp8,0,0.28564479351043703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,32,128,1,bfloat16,bfloat16,0,0.19277759790420532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,8,128,1,fp8,fp8,0,0.2821264028549194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,32,128,1,bfloat16,fp8,0,0.16100800037384033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,32,128,1,fp8,fp8,0,0.16392639875411988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,1,128,1,bfloat16,fp8,0,0.14268640279769898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,1,128,1,fp8,fp8,0,0.14558399915695192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,2,128,1,bfloat16,bfloat16,0,0.15257439613342286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,2,128,1,bfloat16,fp8,0,0.14350559711456298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,2,128,1,fp8,fp8,0,0.14654879570007323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,4,128,1,bfloat16,bfloat16,0,0.1536080002784729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,4,128,1,bfloat16,fp8,0,0.14498720169067383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,4,128,1,fp8,fp8,0,0.14493440389633178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,8,128,1,bfloat16,bfloat16,0,0.15632640123367308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,8,128,1,bfloat16,fp8,0,0.14387840032577515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,32,32,128,1,bfloat16,fp8,0,0.6478079795837403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,8,128,1,fp8,fp8,0,0.14666240215301513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,32,32,128,1,bfloat16,bfloat16,0,0.35970559120178225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,32,1,128,1,bfloat16,bfloat16,0,1.2178383827209474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,32,1,128,1,bfloat16,bfloat16,0,0.15209439992904664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,1,128,1,bfloat16,fp8,0,4.076887893676758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,1,128,1,fp8,fp8,0,4.149095916748047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,1,128,1,bfloat16,bfloat16,0,4.617244720458984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,2,128,1,bfloat16,bfloat16,0,4.699927902221679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,2,128,1,bfloat16,fp8,0,4.11577262878418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,2,128,1,fp8,fp8,0,4.1175281524658205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,4,128,1,bfloat16,bfloat16,0,4.778126525878906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,4,128,1,bfloat16,fp8,0,4.125572967529297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,4,128,1,fp8,fp8,0,4.120251083374024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,32,128,1,bfloat16,bfloat16,0,2.8773967742919924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,8,128,1,bfloat16,fp8,0,4.28883056640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,8,128,1,bfloat16,bfloat16,0,4.885307312011719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,32,128,1,bfloat16,fp8,0,2.4522560119628904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,32,8,128,1,fp8,fp8,0,4.202995300292969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,32,128,1,fp8,fp8,0,2.556324768066406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,1,128,1,bfloat16,bfloat16,0,2.1915088653564454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,1,128,1,bfloat16,fp8,0,2.026865577697754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,1,128,1,fp8,fp8,0,2.1756784439086916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,2,128,1,fp8,fp8,0,2.051169586181641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,2,128,1,bfloat16,bfloat16,0,2.3767744064331056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,2,128,1,bfloat16,fp8,0,2.0862560272216797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,4,128,1,bfloat16,bfloat16,0,2.2099952697753906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,4,128,1,bfloat16,fp8,0,2.0780336380004885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,4,128,1,fp8,fp8,0,2.104812812805176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,8,128,1,bfloat16,bfloat16,0,2.3894575119018553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,32,128,1,bfloat16,bfloat16,0,1.520207977294922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,8,128,1,bfloat16,fp8,0,2.129715156555176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,32,128,1,fp8,fp8,0,1.351360034942627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,32,8,128,1,fp8,fp8,0,2.111311912536621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,1,128,1,bfloat16,bfloat16,0,1.0920448303222656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,1,128,1,bfloat16,fp8,0,1.0282064437866212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,1,128,1,fp8,fp8,0,1.1381440162658691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,2,128,1,bfloat16,bfloat16,0,1.1594079971313476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,2,128,1,bfloat16,fp8,0,1.0313712120056153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,2,128,1,fp8,fp8,0,1.0936495780944824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,4,128,1,bfloat16,fp8,0,1.0732687950134276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,4,128,1,bfloat16,bfloat16,0,1.1297231674194337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,4,128,1,fp8,fp8,0,1.090123176574707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,8,128,1,bfloat16,bfloat16,0,1.2165519714355468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,1,128,1,bfloat16,bfloat16,0,0.5554175853729248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,32,128,1,bfloat16,fp8,0,0.6494272232055665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,32,128,1,bfloat16,bfloat16,0,0.725764799118042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,8,128,1,fp8,fp8,0,1.0614607810974122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,1,128,1,bfloat16,fp8,0,0.5434512138366699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,1,128,1,fp8,fp8,0,0.5325168132781982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,2,128,1,bfloat16,bfloat16,0,0.5657248020172119
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,2,128,1,bfloat16,fp8,0,0.5355231761932373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,2,128,1,fp8,fp8,0,0.5159215927124023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,32,128,1,bfloat16,fp8,0,1.2221471786499023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,4,128,1,bfloat16,bfloat16,0,0.5726592063903808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,4,128,1,bfloat16,fp8,0,0.5374080181121826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,4,128,1,fp8,fp8,0,0.5405344009399414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,8,128,1,bfloat16,bfloat16,0,0.5880352020263672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,8,128,1,bfloat16,fp8,0,0.5315584182739258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,8,128,1,fp8,fp8,0,0.5371056079864502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,32,128,1,bfloat16,fp8,0,0.32289440631866456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,32,128,1,bfloat16,bfloat16,0,0.36917600631713865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,32,128,1,fp8,fp8,0,0.32299199104309084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,1,128,1,bfloat16,bfloat16,0,0.28113760948181155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,1,128,1,bfloat16,fp8,0,0.26295199394226076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,2,128,1,bfloat16,bfloat16,0,0.2837376117706299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,1,128,1,fp8,fp8,0,0.26582560539245603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,2,128,1,bfloat16,fp8,0,0.2716624021530151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,4,128,1,bfloat16,bfloat16,0,0.28947839736938474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,4,128,1,fp8,fp8,0,0.2748703956604004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,8,128,1,bfloat16,bfloat16,0,0.2952752113342285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,8,128,1,fp8,fp8,0,0.2804640054702759
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,32,128,1,bfloat16,fp8,0,0.1648959994316101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,32,128,1,bfloat16,bfloat16,0,0.19320640563964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,32,128,1,fp8,fp8,0,0.1677616000175476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,1,128,1,bfloat16,bfloat16,0,0.14357279539108275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,1,128,1,bfloat16,fp8,0,0.14125440120697022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,1,128,1,fp8,fp8,0,0.14122400283813477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,2,128,1,bfloat16,bfloat16,0,0.14503040313720703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,2,128,1,bfloat16,fp8,0,0.1422111988067627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,2,128,1,fp8,fp8,0,0.14211360216140748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,4,128,1,bfloat16,bfloat16,0,0.14803680181503295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,4,128,1,bfloat16,fp8,0,0.14332959651947022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,32,32,128,1,fp8,fp8,0,0.6513120174407959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,4,128,1,fp8,fp8,0,0.14200799465179442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,32,8,128,1,bfloat16,fp8,0,1.1828895568847657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,8,128,1,fp8,fp8,0,0.1464192032814026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,8,128,1,bfloat16,fp8,0,0.14369759559631348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,32,128,1,bfloat16,bfloat16,0,0.10036159753799438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,32,128,1,bfloat16,fp8,0,0.09028480052947999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,1,128,1,bfloat16,bfloat16,0,0.07449600100517273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,1,128,1,bfloat16,fp8,0,0.07297120094299317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,1,128,1,fp8,fp8,0,0.07142559885978698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,2,128,1,bfloat16,bfloat16,0,0.07807999849319458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,2,128,1,bfloat16,fp8,0,0.07141759991645813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,2,128,1,fp8,fp8,0,0.072707200050354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,4,128,1,bfloat16,bfloat16,0,0.08033599853515624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,4,128,1,bfloat16,fp8,0,0.07347679734230042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,4,128,1,fp8,fp8,0,0.07173439860343933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,8,128,1,bfloat16,bfloat16,0,0.08347359895706177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,8,128,1,bfloat16,fp8,0,0.07333920001983643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,8,128,1,fp8,fp8,0,0.07471200227737426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,2,128,1,fp8,fp8,0,0.2701375961303711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,4,128,1,bfloat16,fp8,0,0.26621921062469484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,32,8,128,1,bfloat16,fp8,0,0.2795104026794434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,32,8,128,1,bfloat16,bfloat16,0,0.1565071940422058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,32,32,128,1,fp8,fp8,0,0.08862720131874084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,1,128,1,bfloat16,fp8,0,2.5800352096557617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,1,128,1,fp8,fp8,0,2.5723215103149415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,2,128,1,bfloat16,bfloat16,0,2.690446472167969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,2,128,1,bfloat16,fp8,0,2.5923200607299806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,2,128,1,fp8,fp8,0,2.5764959335327147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,4,128,1,bfloat16,fp8,0,2.6052751541137695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,4,128,1,bfloat16,bfloat16,0,2.8002368927001955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,4,128,1,fp8,fp8,0,2.5827423095703126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,1,128,1,bfloat16,bfloat16,0,2.7200368881225585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,8,128,1,bfloat16,fp8,0,2.6324623107910154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,32,128,1,bfloat16,fp8,0,1.6879312515258789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,32,128,1,bfloat16,bfloat16,0,1.8351615905761718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,1,128,1,bfloat16,bfloat16,0,1.3198863983154296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,32,128,1,fp8,fp8,0,1.587292766571045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,8,128,1,fp8,fp8,0,2.641097640991211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,1,128,1,fp8,fp8,0,1.2680975914001464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,2,128,1,bfloat16,bfloat16,0,1.3277456283569335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,2,128,1,fp8,fp8,0,1.355183982849121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,2,128,1,bfloat16,fp8,0,1.3192288398742675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,4,128,1,bfloat16,bfloat16,0,1.3666336059570312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,4,128,1,bfloat16,fp8,0,1.288649559020996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,4,128,1,fp8,fp8,0,1.2779696464538575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,32,8,128,1,bfloat16,bfloat16,0,2.9829584121704102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,32,128,1,bfloat16,fp8,0,0.7923776149749756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,32,128,1,fp8,fp8,0,0.7895408153533936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,32,128,1,bfloat16,bfloat16,0,0.937548828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,8,128,1,bfloat16,bfloat16,0,1.4603967666625977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,8,128,1,fp8,fp8,0,1.3576704025268556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,1,128,1,bfloat16,fp8,0,0.6373040199279785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,1,128,1,bfloat16,bfloat16,0,0.6604703903198242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,1,128,1,fp8,fp8,0,0.6350527763366699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,2,128,1,bfloat16,bfloat16,0,0.6850272178649902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,2,128,1,fp8,fp8,0,0.6513711929321289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,2,128,1,bfloat16,fp8,0,0.6883632183074951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,1,128,1,bfloat16,fp8,0,1.2930527687072755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,4,128,1,bfloat16,bfloat16,0,0.6776559829711915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,4,128,1,fp8,fp8,0,0.6458240032196045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,4,128,1,bfloat16,fp8,0,0.6570767879486084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,8,128,1,bfloat16,bfloat16,0,0.7346384048461914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,32,128,1,bfloat16,bfloat16,0,0.46991519927978515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,8,128,1,bfloat16,fp8,0,0.674455976486206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,32,8,128,1,fp8,fp8,0,0.7111311912536621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,32,128,1,bfloat16,fp8,0,0.40335841178894044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,32,128,1,fp8,fp8,0,0.40236477851867675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,1,128,1,bfloat16,fp8,0,0.3342672109603882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,1,128,1,fp8,fp8,0,0.3269792079925537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,2,128,1,bfloat16,bfloat16,0,0.3425568103790283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,2,128,1,bfloat16,fp8,0,0.34025280475616454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,2,128,1,fp8,fp8,0,0.3254143953323364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,4,128,1,bfloat16,bfloat16,0,0.35227839946746825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,4,128,1,bfloat16,fp8,0,0.33724639415740965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,4,128,1,fp8,fp8,0,0.33281760215759276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,8,128,1,bfloat16,bfloat16,0,0.36767361164093015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,8,128,1,bfloat16,fp8,0,0.3484447956085205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,32,128,1,bfloat16,bfloat16,0,0.24436159133911134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,8,128,1,fp8,fp8,0,0.3398688077926636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,32,128,1,bfloat16,fp8,0,0.2131824016571045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,32,128,1,fp8,fp8,0,0.21064159870147706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,1,128,1,bfloat16,bfloat16,0,0.17640960216522217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,1,128,1,bfloat16,fp8,0,0.17293599843978882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,2,128,1,bfloat16,bfloat16,0,0.17745920419692993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,2,128,1,bfloat16,fp8,0,0.17249759435653686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,2,128,1,fp8,fp8,0,0.17285120487213135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,4,128,1,bfloat16,fp8,0,0.17587039470672608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,4,128,1,fp8,fp8,0,0.1740031957626343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,8,128,1,bfloat16,bfloat16,0,0.1897279977798462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,8,128,1,bfloat16,fp8,0,0.1789039969444275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,8,128,1,fp8,fp8,0,0.1802559971809387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,32,128,1,bfloat16,fp8,0,0.11399519443511963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,32,128,1,bfloat16,bfloat16,0,0.12974879741668702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,32,128,1,fp8,fp8,0,0.1140671968460083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,1,128,1,bfloat16,bfloat16,0,0.09371039867401124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,32,8,128,1,bfloat16,fp8,0,1.3662096023559571
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,1,128,1,bfloat16,fp8,0,0.0910095989704132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,2,128,1,bfloat16,fp8,0,0.09154239892959595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,2,128,1,fp8,fp8,0,0.09168000221252441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,4,128,1,bfloat16,bfloat16,0,0.09700480103492737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,4,128,1,bfloat16,fp8,0,0.093259197473526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,4,128,1,fp8,fp8,0,0.09337120056152344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,8,128,1,bfloat16,bfloat16,0,0.10147360563278199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,8,128,1,bfloat16,fp8,0,0.09491199851036072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,8,128,1,fp8,fp8,0,0.09542239904403686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,32,128,1,bfloat16,bfloat16,0,0.07054399847984313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,32,128,1,bfloat16,fp8,0,0.06178879737854004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,32,128,1,fp8,fp8,0,0.06228320002555847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,1,128,1,bfloat16,bfloat16,0,0.04935519993305206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,1,128,1,bfloat16,fp8,0,0.048635199666023254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,1,128,1,fp8,fp8,0,0.04835520088672638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,2,128,1,bfloat16,bfloat16,0,0.04968160092830658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,2,128,1,bfloat16,fp8,0,0.04843200147151947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,2,128,1,fp8,fp8,0,0.04858559966087341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,4,128,1,bfloat16,bfloat16,0,0.05031520128250122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,4,128,1,bfloat16,fp8,0,0.04885759949684143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,4,128,1,fp8,fp8,0,0.04885919988155365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,8,128,1,bfloat16,bfloat16,0,0.05437920093536377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,8,128,1,bfloat16,fp8,0,0.04914399981498718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,32,1,128,1,bfloat16,bfloat16,0,0.3364192008972168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,32,8,128,1,fp8,fp8,0,0.04945439994335175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,1,128,1,fp8,fp8,0,0.17188479900360107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,32,4,128,1,bfloat16,bfloat16,0,0.18290560245513915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,2,128,1,bfloat16,bfloat16,0,0.09500960111618043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,1,128,1,bfloat16,fp8,0,2.6549072265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,1,128,1,bfloat16,bfloat16,0,2.8012800216674805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,1,128,1,fp8,fp8,0,2.692483139038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,2,128,1,bfloat16,bfloat16,0,2.793294334411621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,2,128,1,bfloat16,fp8,0,2.6800079345703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,2,128,1,fp8,fp8,0,2.6576976776123047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,32,1,128,1,fp8,fp8,0,0.09088159799575805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,4,128,1,bfloat16,fp8,0,2.698734474182129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,4,128,1,bfloat16,bfloat16,0,2.8571855545043947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,4,128,1,fp8,fp8,0,2.6963903427124025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,8,128,1,bfloat16,fp8,0,2.766856002807617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,8,128,1,bfloat16,bfloat16,0,3.088999938964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,1,128,1,bfloat16,fp8,0,1.3150768280029297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,1,128,1,bfloat16,bfloat16,0,1.3876848220825195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,32,8,128,1,fp8,fp8,0,2.7789775848388674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,32,128,1,bfloat16,fp8,0,1.7409791946411133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,32,128,1,bfloat16,bfloat16,0,2.0597728729248046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,32,128,1,fp8,fp8,0,1.7214496612548829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,1,128,1,fp8,fp8,0,1.3221535682678223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,2,128,1,bfloat16,fp8,0,1.3189536094665528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,2,128,1,bfloat16,bfloat16,0,1.3780976295471192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,2,128,1,fp8,fp8,0,1.350337600708008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,4,128,1,bfloat16,fp8,0,1.355795192718506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,4,128,1,bfloat16,bfloat16,0,1.4310272216796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,4,128,1,fp8,fp8,0,1.361524772644043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,8,128,1,bfloat16,bfloat16,0,1.505940818786621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,32,128,1,bfloat16,fp8,0,0.8717568397521973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,32,128,1,bfloat16,bfloat16,0,1.0420928001403809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,8,128,1,bfloat16,fp8,0,1.3897263526916503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,1,128,1,bfloat16,fp8,0,0.6737071990966796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,1,128,1,bfloat16,bfloat16,0,0.710374402999878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,32,128,1,fp8,fp8,0,0.8751328468322754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,32,8,128,1,fp8,fp8,0,1.3821999549865722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,1,128,1,fp8,fp8,0,0.7564367771148681
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,2,128,1,bfloat16,bfloat16,0,0.6886623859405517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,2,128,1,bfloat16,fp8,0,0.6721343994140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,4,128,1,bfloat16,bfloat16,0,0.7218160152435302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,4,128,1,bfloat16,fp8,0,0.7046383857727051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,4,128,1,fp8,fp8,0,0.6848400115966797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,8,128,1,bfloat16,bfloat16,0,0.7520016193389892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,8,128,1,bfloat16,fp8,0,0.744649600982666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,32,128,1,bfloat16,bfloat16,0,0.5211376190185547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,8,128,1,fp8,fp8,0,0.6982960224151611
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,32,128,1,bfloat16,fp8,0,0.44533600807189944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,32,128,1,fp8,fp8,0,0.4445487976074219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,1,128,1,bfloat16,fp8,0,0.34069440364837644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,1,128,1,fp8,fp8,0,0.3537359952926636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,2,128,1,bfloat16,bfloat16,0,0.3490031957626343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,2,128,1,bfloat16,fp8,0,0.34516799449920654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,2,128,1,fp8,fp8,0,0.34387199878692626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,4,128,1,bfloat16,bfloat16,0,0.36332480907440184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,4,128,1,bfloat16,fp8,0,0.3485584020614624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,4,128,1,fp8,fp8,0,0.35639679431915283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,32,128,1,bfloat16,bfloat16,0,0.2704463958740234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,32,128,1,bfloat16,fp8,0,0.2287839889526367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,8,128,1,fp8,fp8,0,0.3607712030410767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,8,128,1,bfloat16,fp8,0,0.36272320747375486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,32,128,1,fp8,fp8,0,0.2333695888519287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,1,128,1,bfloat16,bfloat16,0,0.1792863965034485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,1,128,1,fp8,fp8,0,0.17703039646148683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,2,128,1,bfloat16,fp8,0,0.18131200075149537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,2,128,1,fp8,fp8,0,0.17973599433898926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,4,128,1,bfloat16,bfloat16,0,0.1893056035041809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,4,128,1,bfloat16,fp8,0,0.1840880036354065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,4,128,1,fp8,fp8,0,0.18222880363464355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,8,128,1,bfloat16,bfloat16,0,0.19985439777374267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,8,128,1,bfloat16,fp8,0,0.18988159894943238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,32,128,1,bfloat16,bfloat16,0,0.1423200011253357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,32,128,1,bfloat16,fp8,0,0.12101600170135499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,32,128,1,fp8,fp8,0,0.1217695951461792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,1,128,1,bfloat16,bfloat16,0,0.09494559764862061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,1,128,1,bfloat16,fp8,0,0.09429600238800048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,1,128,1,fp8,fp8,0,0.09333919882774352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,2,128,1,bfloat16,bfloat16,0,0.0964576005935669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,2,128,1,bfloat16,fp8,0,0.09431520104408264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,32,2,128,1,fp8,fp8,0,0.669547176361084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,2,128,1,fp8,fp8,0,0.09394720196723938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,4,128,1,bfloat16,bfloat16,0,0.09820160269737244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,4,128,1,bfloat16,fp8,0,0.09566239714622497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,4,128,1,fp8,fp8,0,0.09548959732055665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,8,128,1,bfloat16,bfloat16,0,0.1048624038696289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,8,128,1,bfloat16,fp8,0,0.0981231987476349
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,1,128,1,bfloat16,bfloat16,0,0.3488976001739502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,32,128,1,bfloat16,bfloat16,0,0.07574880123138428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,32,128,1,bfloat16,fp8,0,0.06681439876556397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,32,128,1,fp8,fp8,0,0.06609920263290406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,1,128,1,bfloat16,bfloat16,0,0.05082240104675293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,1,128,1,bfloat16,fp8,0,0.04929440021514893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,1,128,1,fp8,fp8,0,0.05001119971275329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,2,128,1,bfloat16,bfloat16,0,0.05121920108795166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,2,128,1,bfloat16,fp8,0,0.050046402215957644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,2,128,1,fp8,fp8,0,0.04956640005111694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,4,128,1,bfloat16,bfloat16,0,0.05551360249519348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,4,128,1,bfloat16,fp8,0,0.04931040108203888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,4,128,1,fp8,fp8,0,0.05007839798927307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,8,128,1,bfloat16,bfloat16,0,0.057550400495529175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,32,8,128,1,bfloat16,bfloat16,0,0.38128960132598877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,8,128,1,bfloat16,fp8,0,0.05234559774398804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,32,8,128,1,fp8,fp8,0,0.051239997148513794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,32,128,1,bfloat16,bfloat16,0,0.042361599206924436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,32,128,1,bfloat16,fp8,0,0.03355199992656708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,32,128,1,fp8,fp8,0,0.03295679986476898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,1,128,1,bfloat16,bfloat16,0,0.02951200008392334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,1,128,1,bfloat16,fp8,0,0.17584160566329957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,1,128,1,bfloat16,fp8,0,0.029926401376724244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,2,128,1,bfloat16,bfloat16,0,0.18438559770584106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,1,128,1,fp8,fp8,0,0.02935839891433716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,2,128,1,bfloat16,bfloat16,0,0.029363200068473816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,2,128,1,bfloat16,fp8,0,0.030193600058555602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,2,128,1,fp8,fp8,0,0.029937601089477538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,4,128,1,bfloat16,bfloat16,0,0.029550400376319886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,4,128,1,bfloat16,fp8,0,0.029600000381469725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,4,128,1,fp8,fp8,0,0.030211201310157774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,8,128,1,bfloat16,bfloat16,0,0.03019680082798004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,8,128,1,bfloat16,fp8,0,0.03039039969444275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,32,8,128,1,fp8,fp8,0,0.030126398801803587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,32,8,128,1,fp8,fp8,0,0.1881343960762024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,1,128,1,bfloat16,bfloat16,0,1.89563045501709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,32,8,128,1,fp8,fp8,0,0.09813119769096375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,2,128,1,bfloat16,bfloat16,0,1.947947120666504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,1,128,1,bfloat16,fp8,0,1.9491312026977539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,2,128,1,bfloat16,fp8,0,1.9397424697875976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,2,128,1,fp8,fp8,0,1.9373584747314454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,4,128,1,bfloat16,bfloat16,0,2.0673263549804686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,4,128,1,bfloat16,fp8,0,1.9834672927856445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,4,128,1,fp8,fp8,0,1.9906095504760741
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,8,128,1,bfloat16,fp8,0,2.067056083679199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,8,128,1,bfloat16,bfloat16,0,2.244024085998535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,8,128,1,fp8,fp8,0,2.0675888061523438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,32,1,128,1,fp8,fp8,0,1.9591424942016602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,1,128,1,bfloat16,bfloat16,0,0.9538144111633301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,1,128,1,bfloat16,fp8,0,0.9755871772766114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,1,128,1,fp8,fp8,0,0.9690367698669433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,32,128,1,bfloat16,fp8,0,1.4007840156555176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,32,128,1,fp8,fp8,0,1.3996800422668456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,2,128,1,bfloat16,bfloat16,0,0.975819206237793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,2,128,1,fp8,fp8,0,0.9735919952392578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,2,128,1,bfloat16,fp8,0,0.9736175537109375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,4,128,1,bfloat16,bfloat16,0,1.0291791915893556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,4,128,1,bfloat16,fp8,0,0.9966752052307128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,4,128,1,fp8,fp8,0,0.9970848083496093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,8,128,1,bfloat16,bfloat16,0,1.1224847793579102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,8,128,1,bfloat16,fp8,0,1.0340080261230469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,1,128,1,bfloat16,bfloat16,0,0.4819727897644043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,32,128,1,bfloat16,fp8,0,0.7072447776794434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,32,128,1,bfloat16,bfloat16,0,0.836854362487793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,32,128,1,fp8,fp8,0,0.7062943935394287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,1,128,1,fp8,fp8,0,0.48815040588378905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,1,128,1,bfloat16,fp8,0,0.49057278633117674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,8,128,1,fp8,fp8,0,1.040329647064209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,2,128,1,bfloat16,bfloat16,0,0.49218077659606935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,2,128,1,bfloat16,fp8,0,0.4946415901184082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,4,128,1,bfloat16,bfloat16,0,0.5177984237670898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,4,128,1,bfloat16,fp8,0,0.502291202545166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,2,128,1,fp8,fp8,0,0.4934271812438965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,4,128,1,fp8,fp8,0,0.5131264209747315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,8,128,1,bfloat16,bfloat16,0,0.5683135986328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,8,128,1,bfloat16,fp8,0,0.5212495803833008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,1,128,1,bfloat16,bfloat16,0,0.2524800062179565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,32,128,1,bfloat16,bfloat16,0,0.42529921531677245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,32,128,1,bfloat16,fp8,0,0.35818240642547605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,32,8,128,1,fp8,fp8,0,0.5278096199035645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,1,128,1,bfloat16,fp8,0,0.2508752107620239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,1,128,1,fp8,fp8,0,0.24863998889923095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,2,128,1,bfloat16,fp8,0,0.25048160552978516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,2,128,1,fp8,fp8,0,0.252620792388916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,4,128,1,bfloat16,bfloat16,0,0.2688976049423218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,4,128,1,bfloat16,fp8,0,0.26243679523468016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,32,32,128,1,bfloat16,bfloat16,0,1.6800559997558593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,4,128,1,fp8,fp8,0,0.2568928003311157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,8,128,1,bfloat16,bfloat16,0,0.28949120044708254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,32,128,1,bfloat16,bfloat16,0,0.22347040176391603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,8,128,1,bfloat16,fp8,0,0.27124159336090087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,8,128,1,fp8,fp8,0,0.27307519912719724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,32,128,1,fp8,fp8,0,0.18523679971694945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,32,128,1,bfloat16,fp8,0,0.19921599626541137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,1,128,1,bfloat16,bfloat16,0,0.12948800325393678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,1,128,1,bfloat16,fp8,0,0.13238719701766968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,1,128,1,fp8,fp8,0,0.13305599689483644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,2,128,1,bfloat16,bfloat16,0,0.13211679458618164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,2,128,1,bfloat16,fp8,0,0.13482719659805298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,2,128,1,fp8,fp8,0,0.13069759607315062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,4,128,1,bfloat16,bfloat16,0,0.14137760400772095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,4,128,1,bfloat16,fp8,0,0.13664159774780274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,4,128,1,fp8,fp8,0,0.13394240140914918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,8,128,1,bfloat16,bfloat16,0,0.1535215973854065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,32,128,1,bfloat16,fp8,0,0.09850239753723145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,32,128,1,bfloat16,bfloat16,0,0.11822880506515503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,32,128,1,fp8,fp8,0,0.10040960311889649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,1,128,1,bfloat16,bfloat16,0,0.07150400280952454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,1,128,1,bfloat16,fp8,0,0.07269440293312072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,1,128,1,fp8,fp8,0,0.07262880206108094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,2,128,1,bfloat16,bfloat16,0,0.07297599911689759
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,2,128,1,bfloat16,fp8,0,0.07296000123023987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,2,128,1,fp8,fp8,0,0.07347040176391602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,4,128,1,bfloat16,bfloat16,0,0.07670400142669678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,4,128,1,bfloat16,fp8,0,0.07451999783515931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,4,128,1,fp8,fp8,0,0.07480480074882508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,8,128,1,bfloat16,bfloat16,0,0.08342400193214417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,8,128,1,bfloat16,fp8,0,0.07777119874954223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,32,128,1,bfloat16,bfloat16,0,0.06534240245819092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,32,128,1,bfloat16,fp8,0,0.05727360248565674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,32,128,1,fp8,fp8,0,0.05830879807472229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,1,128,1,bfloat16,bfloat16,0,0.038996800780296326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,1,128,1,bfloat16,fp8,0,0.0383296012878418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,1,128,1,fp8,fp8,0,0.03829439878463745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,2,128,1,bfloat16,bfloat16,0,0.040171200037002565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,2,128,1,bfloat16,fp8,0,0.03841440081596374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,2,128,1,fp8,fp8,0,0.03842880129814148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,4,128,1,bfloat16,bfloat16,0,0.04313279986381531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,4,128,1,bfloat16,fp8,0,0.0389167994260788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,4,128,1,fp8,fp8,0,0.038940799236297605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,8,128,1,bfloat16,fp8,0,0.04038879871368408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,8,128,1,fp8,fp8,0,0.04048320055007935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,32,128,1,bfloat16,bfloat16,0,0.03789600133895874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,32,128,1,fp8,fp8,0,0.3600879907608032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,32,128,1,bfloat16,fp8,0,0.03128960132598877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,32,128,1,fp8,fp8,0,0.030483201146125793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,1,128,1,bfloat16,bfloat16,0,0.022614400088787078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,32,2,128,1,bfloat16,bfloat16,0,0.2592128038406372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,1,128,1,bfloat16,fp8,0,0.023844799399375914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,1,128,1,fp8,fp8,0,0.02406879961490631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,2,128,1,bfloat16,bfloat16,0,0.022944000363349915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,2,128,1,fp8,fp8,0,0.024062399566173554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,4,128,1,bfloat16,bfloat16,0,0.023324799537658692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,4,128,1,bfloat16,fp8,0,0.02447039932012558
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,4,128,1,fp8,fp8,0,0.024273599684238433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,8,128,1,bfloat16,bfloat16,0,0.023841600120067596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,8,128,1,bfloat16,fp8,0,0.02457599937915802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,8,128,1,fp8,fp8,0,0.02465440034866333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,32,128,1,bfloat16,bfloat16,0,0.020716799795627593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,32,128,1,bfloat16,fp8,0,0.020561599731445314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,32,128,1,fp8,fp8,0,0.020457600057125092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,1,128,1,bfloat16,bfloat16,0,0.017723199725151063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,1,128,1,bfloat16,fp8,0,0.018915200233459474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,1,128,1,fp8,fp8,0,0.018958400189876556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,2,128,1,bfloat16,bfloat16,0,0.017844800651073457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,2,128,1,bfloat16,fp8,0,0.018987199664115904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,2,128,1,fp8,fp8,0,0.018904000520706177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,4,128,1,bfloat16,bfloat16,0,0.018113599717617036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,4,128,1,bfloat16,fp8,0,0.01907680034637451
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,8,128,1,bfloat16,fp8,0,0.13929439783096315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,4,128,1,fp8,fp8,0,0.019152000546455383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,8,128,1,bfloat16,bfloat16,0,0.018401600420475006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,32,8,128,1,fp8,fp8,0,0.14182560443878173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,8,128,1,bfloat16,fp8,0,0.01924159973859787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,32,8,128,1,fp8,fp8,0,0.01918559968471527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,32,8,128,1,fp8,fp8,0,0.07777919769287109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,1,128,1,bfloat16,bfloat16,0,0.7508975982666015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,2,128,1,bfloat16,bfloat16,0,0.7779263973236084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,1,128,1,fp8,fp8,0,0.7989535808563233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,32,8,128,1,bfloat16,bfloat16,0,0.045388799905776975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,2,128,1,bfloat16,fp8,0,0.7925024032592773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,2,128,1,fp8,fp8,0,0.791815996170044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,4,128,1,bfloat16,bfloat16,0,0.8230575561523438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,32,2,128,1,bfloat16,fp8,0,0.023871999979019166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,4,128,1,fp8,fp8,0,0.8155535697937012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,8,128,1,bfloat16,bfloat16,0,0.9087535858154296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,8,128,1,bfloat16,fp8,0,0.8590784072875977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,8,128,1,fp8,fp8,0,0.859124755859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,32,128,1,bfloat16,bfloat16,0,0.7701087951660156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,32,128,1,bfloat16,fp8,0,0.5983471870422363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,1,128,1,bfloat16,bfloat16,0,0.3824912071228027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,32,128,1,fp8,fp8,0,0.5983488082885742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,1,128,1,bfloat16,fp8,0,0.4001632213592529
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,1,128,1,fp8,fp8,0,0.40014400482177737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,2,128,1,bfloat16,bfloat16,0,0.39365758895874026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,2,128,1,bfloat16,fp8,0,0.40354080200195314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,2,128,1,fp8,fp8,0,0.4032656192779541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,4,128,1,bfloat16,bfloat16,0,0.41584000587463377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,4,128,1,bfloat16,fp8,0,0.41497278213500977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,4,128,1,fp8,fp8,0,0.4150832176208496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,8,128,1,bfloat16,bfloat16,0,0.4622047901153564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,8,128,1,fp8,fp8,0,0.43659520149230957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,32,128,1,bfloat16,fp8,0,0.30702240467071534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,32,128,1,bfloat16,bfloat16,0,0.39227519035339353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,1,128,1,bfloat16,fp8,0,0.7872960090637207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,1,128,1,bfloat16,bfloat16,0,0.19862719774246215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,1,128,1,bfloat16,fp8,0,0.20635039806365968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,32,128,1,fp8,fp8,0,0.3070496082305908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,1,128,1,fp8,fp8,0,0.2067552089691162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,2,128,1,bfloat16,bfloat16,0,0.2048367977142334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,2,128,1,bfloat16,fp8,0,0.20840001106262207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,2,128,1,fp8,fp8,0,0.20740480422973634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,4,128,1,bfloat16,bfloat16,0,0.2152928113937378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,4,128,1,bfloat16,fp8,0,0.2143471956253052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,4,128,1,fp8,fp8,0,0.2149280071258545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,8,128,1,bfloat16,bfloat16,0,0.23780479431152343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,8,128,1,bfloat16,fp8,0,0.2241231918334961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,32,8,128,1,fp8,fp8,0,0.22580959796905517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,32,128,1,bfloat16,bfloat16,0,0.20414400100708008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,32,128,1,bfloat16,fp8,0,0.1607375979423523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,32,4,128,1,bfloat16,fp8,0,0.8159328460693359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,1,128,1,bfloat16,bfloat16,0,0.10591360330581664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,1,128,1,bfloat16,fp8,0,0.10961120128631592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,1,128,1,fp8,fp8,0,0.1098031997680664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,32,128,1,fp8,fp8,0,0.1601040005683899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,2,128,1,bfloat16,bfloat16,0,0.10927040576934814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,2,128,1,fp8,fp8,0,0.11000959873199463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,2,128,1,bfloat16,fp8,0,0.11070560216903687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,4,128,1,bfloat16,bfloat16,0,0.11509920358657837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,8,128,1,bfloat16,bfloat16,0,0.12629760503768922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,8,128,1,bfloat16,fp8,0,0.11887520551681519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,8,128,1,fp8,fp8,0,0.11975359916687012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,32,128,1,bfloat16,fp8,0,0.08744959831237793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,32,128,1,bfloat16,bfloat16,0,0.10947359800338745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,1,128,1,bfloat16,bfloat16,0,0.05995519757270813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,32,128,1,fp8,fp8,0,0.08653920292854309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,1,128,1,bfloat16,fp8,0,0.061136001348495485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,1,128,1,fp8,fp8,0,0.06065599918365479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,2,128,1,bfloat16,bfloat16,0,0.06117600202560425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,2,128,1,bfloat16,fp8,0,0.06089760065078735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,2,128,1,fp8,fp8,0,0.06149759888648987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,4,128,1,bfloat16,bfloat16,0,0.06324160099029541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,4,128,1,bfloat16,fp8,0,0.06308000087738037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,8,128,1,bfloat16,bfloat16,0,0.06907359957695007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,8,128,1,bfloat16,fp8,0,0.06539199948310852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,8,128,1,fp8,fp8,0,0.06556159853935242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,32,128,1,bfloat16,bfloat16,0,0.06110079884529114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,32,128,1,bfloat16,fp8,0,0.049886399507522584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,32,128,1,fp8,fp8,0,0.050323200225830075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,1,128,1,bfloat16,bfloat16,0,0.03285439908504486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,1,128,1,bfloat16,fp8,0,0.03286559879779816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,2,128,1,bfloat16,bfloat16,0,0.034585601091384886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,2,128,1,bfloat16,fp8,0,0.0328031986951828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,2,128,1,fp8,fp8,0,0.03298879861831665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,4,128,1,bfloat16,bfloat16,0,0.03710080087184906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,4,128,1,bfloat16,fp8,0,0.033452799916267394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,4,128,1,fp8,fp8,0,0.033139199018478394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,8,128,1,bfloat16,bfloat16,0,0.04102559983730316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,8,128,1,bfloat16,fp8,0,0.03497759997844696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,8,128,1,fp8,fp8,0,0.03592959940433502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,32,128,1,bfloat16,bfloat16,0,0.03403359949588776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,32,128,1,fp8,fp8,0,0.02547999918460846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,32,8,128,1,bfloat16,fp8,0,0.4369376182556152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,1,128,1,bfloat16,bfloat16,0,0.020718400180339814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,1,128,1,fp8,fp8,0,0.02183839976787567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,2,128,1,bfloat16,bfloat16,0,0.020795199275016784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,2,128,1,bfloat16,fp8,0,0.022071999311447144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,2,128,1,fp8,fp8,0,0.022017599642276765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,4,128,1,bfloat16,bfloat16,0,0.021294400095939636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,4,128,1,bfloat16,fp8,0,0.022180800139904023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,4,128,1,fp8,fp8,0,0.02216159999370575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,8,128,1,bfloat16,bfloat16,0,0.02175839990377426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,8,128,1,bfloat16,fp8,0,0.02231999933719635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,8,128,1,fp8,fp8,0,0.022249600291252135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,32,128,1,bfloat16,fp8,0,0.016470399498939515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,32,128,1,bfloat16,bfloat16,0,0.01703200042247772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,32,128,1,fp8,fp8,0,0.016833600401878358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,1,128,1,bfloat16,bfloat16,0,0.014393599331378936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,1,128,1,bfloat16,fp8,0,0.015332800149917603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,1,128,1,fp8,fp8,0,0.014945599436759948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,2,128,1,bfloat16,bfloat16,0,0.014696000516414643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,2,128,1,bfloat16,fp8,0,0.014947199821472168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,4,128,1,fp8,fp8,0,0.11348799467086793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,2,128,1,fp8,fp8,0,0.01515520066022873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,4,128,1,bfloat16,bfloat16,0,0.014670400321483612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,4,128,1,bfloat16,fp8,0,0.0154448002576828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,4,128,1,fp8,fp8,0,0.015243199467658997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,8,128,1,bfloat16,bfloat16,0,0.014839999377727509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,8,128,1,bfloat16,fp8,0,0.015353600680828094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,32,8,128,1,fp8,fp8,0,0.015240000188350677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,32,128,1,bfloat16,bfloat16,0,0.01555519998073578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,32,128,1,bfloat16,fp8,0,0.015859200060367583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,32,128,1,fp8,fp8,0,0.01578879952430725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,1,128,1,bfloat16,bfloat16,0,0.014084799587726593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,1,128,1,bfloat16,fp8,0,0.015083199739456177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,1,128,1,fp8,fp8,0,0.014921599626541137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,2,128,1,bfloat16,fp8,0,0.014924800395965577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,2,128,1,fp8,fp8,0,0.014929600059986115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,4,128,1,bfloat16,bfloat16,0,0.014721600711345673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,4,128,1,bfloat16,fp8,0,0.015251199901103973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,4,128,1,fp8,fp8,0,0.015057599544525147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,8,128,1,bfloat16,bfloat16,0,0.015240000188350677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,8,128,1,bfloat16,fp8,0,0.015228800475597382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,32,1,128,1,fp8,fp8,0,0.03275040090084076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,8,128,1,fp8,fp8,0,0.015230399370193482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,1,128,1,bfloat16,bfloat16,0,0.37066400051116943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,32,128,1,bfloat16,fp8,0,0.02566240131855011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,32,1,128,1,bfloat16,fp8,0,0.02202720046043396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,1,128,1,fp8,fp8,0,0.3979408025741577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,2,128,1,bfloat16,bfloat16,0,0.3788032054901123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,2,128,1,bfloat16,fp8,0,0.405244779586792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,2,128,1,fp8,fp8,0,0.40395197868347166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,4,128,1,bfloat16,bfloat16,0,0.3982815980911255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,4,128,1,bfloat16,fp8,0,0.42193121910095216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,4,128,1,fp8,fp8,0,0.4200911998748779
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,8,128,1,bfloat16,bfloat16,0,0.4380496025085449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,32,4,128,1,bfloat16,fp8,0,0.1133247971534729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,8,128,1,bfloat16,fp8,0,0.45169920921325685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,8,128,1,fp8,fp8,0,0.4488255977630615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,32,128,1,bfloat16,bfloat16,0,0.3912719964981079
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,32,128,1,bfloat16,fp8,0,0.308732795715332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,32,2,128,1,bfloat16,bfloat16,0,0.014030399918556213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,1,128,1,bfloat16,bfloat16,0,0.1901535987854004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,32,4,128,1,fp8,fp8,0,0.062300801277160645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,32,128,1,fp8,fp8,0,0.310534405708313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,1,128,1,fp8,fp8,0,0.20300159454345704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,2,128,1,bfloat16,bfloat16,0,0.19519840478897094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,2,128,1,bfloat16,fp8,0,0.20689759254455567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,4,128,1,bfloat16,bfloat16,0,0.20650720596313477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,2,128,1,fp8,fp8,0,0.20686399936676025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,4,128,1,bfloat16,fp8,0,0.21574718952178956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,4,128,1,fp8,fp8,0,0.21579198837280272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,8,128,1,bfloat16,bfloat16,0,0.22501759529113768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,8,128,1,bfloat16,fp8,0,0.23065600395202637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,8,128,1,fp8,fp8,0,0.22989120483398437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,32,128,1,bfloat16,bfloat16,0,0.2050863981246948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,1,128,1,bfloat16,bfloat16,0,0.10400320291519165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,32,128,1,bfloat16,fp8,0,0.1637712001800537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,1,128,1,bfloat16,fp8,0,0.1082800030708313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,1,128,1,fp8,fp8,0,0.10872639417648315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,2,128,1,bfloat16,bfloat16,0,0.105075204372406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,2,128,1,bfloat16,fp8,0,0.11069120168685913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,2,128,1,fp8,fp8,0,0.1099087953567505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,4,128,1,bfloat16,bfloat16,0,0.10991519689559937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,4,128,1,bfloat16,fp8,0,0.11402560472488403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,4,128,1,fp8,fp8,0,0.11354399919509887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,8,128,1,bfloat16,bfloat16,0,0.12036960124969483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,8,128,1,bfloat16,fp8,0,0.12172000408172608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,8,128,1,fp8,fp8,0,0.12196799516677856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,32,128,1,bfloat16,fp8,0,0.08678399920463561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,32,128,1,fp8,fp8,0,0.08630080223083496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,1,128,1,bfloat16,bfloat16,0,0.05849120020866394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,1,128,1,bfloat16,fp8,0,0.059595197439193726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,1,128,1,fp8,fp8,0,0.05969439744949341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,2,128,1,bfloat16,bfloat16,0,0.059248000383377075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,2,128,1,bfloat16,fp8,0,0.06054880023002625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,2,128,1,fp8,fp8,0,0.060476797819137576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,4,128,1,bfloat16,bfloat16,0,0.061470401287078855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,4,128,1,bfloat16,fp8,0,0.06153759956359863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,4,128,1,fp8,fp8,0,0.061457598209381105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,8,128,1,bfloat16,bfloat16,0,0.06715840101242065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,8,128,1,bfloat16,fp8,0,0.06477280259132386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,8,128,1,fp8,fp8,0,0.06423519849777222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,32,128,1,bfloat16,bfloat16,0,0.05910559892654419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,32,128,1,bfloat16,fp8,0,0.04658240079879761
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,32,128,1,fp8,fp8,0,0.04621120095252991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,1,128,1,bfloat16,fp8,0,0.02964639961719513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,1,128,1,fp8,fp8,0,0.02964639961719513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,2,128,1,bfloat16,bfloat16,0,0.03153280019760132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,2,128,1,bfloat16,fp8,0,0.02967039942741394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,2,128,1,fp8,fp8,0,0.029516801238059998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,4,128,1,bfloat16,bfloat16,0,0.03437759876251221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,4,128,1,bfloat16,fp8,0,0.030324798822402955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,4,128,1,fp8,fp8,0,0.030435198545455934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,8,128,1,bfloat16,bfloat16,0,0.03876639902591705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,8,128,1,bfloat16,fp8,0,0.03170560002326965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,32,128,1,bfloat16,bfloat16,0,0.03221119940280914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,32,128,1,bfloat16,fp8,0,0.021465599536895752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,8,128,1,fp8,fp8,0,0.03184959888458252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,32,128,1,fp8,fp8,0,0.02165919989347458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,1,128,1,bfloat16,bfloat16,0,0.017796799540519714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,1,128,1,bfloat16,fp8,0,0.019068799912929535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,1,128,1,fp8,fp8,0,0.019092799723148347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,2,128,1,bfloat16,bfloat16,0,0.018068799376487733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,2,128,1,bfloat16,fp8,0,0.01910720020532608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,2,128,1,fp8,fp8,0,0.019083200395107268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,4,128,1,bfloat16,bfloat16,0,0.01839359998703003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,4,128,1,bfloat16,fp8,0,0.01929759979248047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,4,128,1,fp8,fp8,0,0.01926880031824112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,8,128,1,bfloat16,bfloat16,0,0.01908160001039505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,8,128,1,bfloat16,fp8,0,0.0195360004901886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,32,8,128,1,fp8,fp8,0,0.019420799612998963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,32,128,1,bfloat16,bfloat16,0,0.01569119989871979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,32,128,1,bfloat16,fp8,0,0.015134400129318238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,32,128,1,fp8,fp8,0,0.014870400726795196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,1,128,1,bfloat16,bfloat16,0,0.013051199913024902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,32,1,128,1,bfloat16,fp8,0,0.20429599285125732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,1,128,1,bfloat16,fp8,0,0.013924799859523773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,1,128,1,fp8,fp8,0,0.013705599308013915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,2,128,1,bfloat16,bfloat16,0,0.012961600720882416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,2,128,1,fp8,fp8,0,0.013911999762058258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,4,128,1,bfloat16,bfloat16,0,0.01348000019788742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,4,128,1,bfloat16,fp8,0,0.013873599469661713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,4,128,1,fp8,fp8,0,0.013803200423717498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,8,128,1,bfloat16,bfloat16,0,0.013763199746608733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,8,128,1,bfloat16,fp8,0,0.01401599943637848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,8,128,1,fp8,fp8,0,0.013926400244235993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,32,128,1,bfloat16,bfloat16,0,0.014327999949455262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,32,128,1,bfloat16,fp8,0,0.01387999951839447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,32,128,1,fp8,fp8,0,0.014006400108337402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,1,128,1,bfloat16,bfloat16,0,0.012796799838542938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,1,128,1,bfloat16,fp8,0,0.013419200479984284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,1,128,1,fp8,fp8,0,0.013625599443912506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,2,128,1,bfloat16,bfloat16,0,0.012928000092506409
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,32,32,128,1,fp8,fp8,0,0.16259360313415527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,2,128,1,fp8,fp8,0,0.013334399461746216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,4,128,1,bfloat16,bfloat16,0,0.012899200618267059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,4,128,1,bfloat16,fp8,0,0.01340160071849823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,32,1,128,1,bfloat16,fp8,0,0.39868640899658203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,4,128,1,fp8,fp8,0,0.013220800459384919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,8,128,1,bfloat16,bfloat16,0,0.013126400113105775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,8,128,1,bfloat16,fp8,0,0.013777600228786468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,8,128,1,fp8,fp8,0,0.013406400382518769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,32,128,1,bfloat16,bfloat16,0,0.013489599525928497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,32,128,1,bfloat16,fp8,0,0.013775999844074249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,32,128,1,fp8,fp8,0,0.013425600528717042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,1,128,1,bfloat16,fp8,0,0.013273599743843078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,1,128,1,fp8,fp8,0,0.01313599944114685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,32,32,128,1,bfloat16,bfloat16,0,0.10962879657745361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,2,128,1,bfloat16,bfloat16,0,0.012723200023174286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,2,128,1,bfloat16,fp8,0,0.012987199425697326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,2,128,1,fp8,fp8,0,0.013235199451446533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,4,128,1,bfloat16,bfloat16,0,0.012943999469280243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,4,128,1,bfloat16,fp8,0,0.013172799348831176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,4,128,1,fp8,fp8,0,0.01318880021572113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,8,128,1,bfloat16,bfloat16,0,0.012980799376964568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,8,128,1,bfloat16,fp8,0,0.013385599851608277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,8,128,1,fp8,fp8,0,0.013153600692749023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,1,128,1,bfloat16,bfloat16,0,0.18941919803619384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,1,128,1,bfloat16,fp8,0,0.20390079021453858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,1,128,1,fp8,fp8,0,0.20341920852661133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,32,1,128,1,bfloat16,bfloat16,0,0.02935200035572052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,2,128,1,bfloat16,bfloat16,0,0.1947999954223633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,2,128,1,bfloat16,fp8,0,0.20664639472961427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,2,128,1,fp8,fp8,0,0.2056720018386841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,4,128,1,bfloat16,bfloat16,0,0.2056175947189331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,4,128,1,bfloat16,fp8,0,0.21336960792541504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,4,128,1,fp8,fp8,0,0.21342079639434813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,8,128,1,bfloat16,bfloat16,0,0.23429279327392577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,8,128,1,bfloat16,fp8,0,0.22811520099639893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,32,8,128,1,fp8,fp8,0,0.229201602935791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,32,128,1,bfloat16,bfloat16,0,0.2464047908782959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,32,128,1,bfloat16,fp8,0,0.23679358959198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,32,128,1,fp8,fp8,0,0.2368623971939087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,1,128,1,bfloat16,fp8,0,0.10864319801330566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,1,128,1,fp8,fp8,0,0.10819200277328492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,2,128,1,bfloat16,bfloat16,0,0.10498080253601075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,2,128,1,bfloat16,fp8,0,0.1096127986907959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,2,128,1,fp8,fp8,0,0.11009759902954101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,32,2,128,1,bfloat16,fp8,0,0.013819199800491334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,4,128,1,bfloat16,bfloat16,0,0.1109328031539917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,4,128,1,bfloat16,fp8,0,0.11350560188293457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,4,128,1,fp8,fp8,0,0.113646399974823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,8,128,1,bfloat16,bfloat16,0,0.12440320253372192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,8,128,1,bfloat16,fp8,0,0.12085599899291992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,8,128,1,fp8,fp8,0,0.12084640264511108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,32,128,1,bfloat16,bfloat16,0,0.1316480040550232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,32,128,1,bfloat16,fp8,0,0.12399200201034546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,1,128,1,bfloat16,bfloat16,0,0.05878880023956299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,32,128,1,fp8,fp8,0,0.12329119443893433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,1,128,1,bfloat16,fp8,0,0.06001920104026794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,1,128,1,fp8,fp8,0,0.06077439785003662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,2,128,1,bfloat16,fp8,0,0.06107519865036011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,2,128,1,fp8,fp8,0,0.06088640093803406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,32,2,128,1,bfloat16,fp8,0,0.013619199395179749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,4,128,1,bfloat16,bfloat16,0,0.0627232015132904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,4,128,1,bfloat16,fp8,0,0.0623088002204895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,8,128,1,bfloat16,bfloat16,0,0.06798400282859803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,8,128,1,bfloat16,fp8,0,0.06607999801635742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,8,128,1,fp8,fp8,0,0.06553120017051697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,32,128,1,bfloat16,bfloat16,0,0.07230719923973083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,32,128,1,bfloat16,fp8,0,0.06493120193481446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,32,128,1,fp8,fp8,0,0.06412960290908813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,32,1,128,1,bfloat16,bfloat16,0,0.012383999675512314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,1,128,1,bfloat16,fp8,0,0.03023360073566437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,1,128,1,fp8,fp8,0,0.02985759973526001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,2,128,1,bfloat16,bfloat16,0,0.030623999238014222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,2,128,1,bfloat16,fp8,0,0.03088639974594116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,2,128,1,fp8,fp8,0,0.030035200715065002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,4,128,1,bfloat16,bfloat16,0,0.0350816011428833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,4,128,1,bfloat16,fp8,0,0.030742400884628297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,4,128,1,fp8,fp8,0,0.0303056001663208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,8,128,1,bfloat16,bfloat16,0,0.03823359906673431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,8,128,1,bfloat16,fp8,0,0.03250559866428375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,8,128,1,fp8,fp8,0,0.031806400418281554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,32,128,1,bfloat16,fp8,0,0.030817601084709167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,32,128,1,fp8,fp8,0,0.03100000023841858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,1,128,1,bfloat16,bfloat16,0,0.018435199558734894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,1,128,1,bfloat16,fp8,0,0.019072000682353974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,1,128,1,fp8,fp8,0,0.019499200582504272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,2,128,1,bfloat16,bfloat16,0,0.01844319999217987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,2,128,1,bfloat16,fp8,0,0.019223999977111817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,2,128,1,fp8,fp8,0,0.01927199959754944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,4,128,1,bfloat16,bfloat16,0,0.018807999789714813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,4,128,1,bfloat16,fp8,0,0.01934240013360977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,4,128,1,fp8,fp8,0,0.019435200095176696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,8,128,1,bfloat16,bfloat16,0,0.019315199553966524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,8,128,1,bfloat16,fp8,0,0.019513599574565887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,8,128,1,fp8,fp8,0,0.019449600577354433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,32,128,1,bfloat16,bfloat16,0,0.018932799994945525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,32,1,128,1,bfloat16,bfloat16,0,0.10344959497451782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,32,128,1,bfloat16,fp8,0,0.018966400623321535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,32,128,1,fp8,fp8,0,0.019094400107860565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,1,128,1,bfloat16,bfloat16,0,0.013155199587345123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,1,128,1,bfloat16,fp8,0,0.014008000493049622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,1,128,1,fp8,fp8,0,0.013920000195503235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,2,128,1,bfloat16,bfloat16,0,0.013075199723243714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,2,128,1,bfloat16,fp8,0,0.014046399295330048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,2,128,1,fp8,fp8,0,0.013982400298118591
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,4,128,1,bfloat16,bfloat16,0,0.013251200318336487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,4,128,1,bfloat16,fp8,0,0.013995200395584106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,4,128,1,fp8,fp8,0,0.014094400405883788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,8,128,1,bfloat16,bfloat16,0,0.013558399677276612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,8,128,1,bfloat16,fp8,0,0.01403679996728897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,32,8,128,1,fp8,fp8,0,0.014108799397945404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,32,128,1,bfloat16,bfloat16,0,0.013630400598049163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,32,128,1,bfloat16,fp8,0,0.013788799941539764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,32,128,1,fp8,fp8,0,0.013872000575065612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,1,128,1,bfloat16,fp8,0,0.013491199910640716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,1,128,1,fp8,fp8,0,0.01356319934129715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,2,128,1,bfloat16,bfloat16,0,0.012708799540996551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,2,128,1,bfloat16,bfloat16,0,0.05953599810600281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,2,128,1,fp8,fp8,0,0.013488000631332398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,4,128,1,bfloat16,bfloat16,0,0.012671999633312225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,4,128,1,bfloat16,fp8,0,0.013662399351596832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,32,4,128,1,fp8,fp8,0,0.06288800239562989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,4,128,1,fp8,fp8,0,0.013252800703048706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,8,128,1,bfloat16,bfloat16,0,0.013209599256515502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,8,128,1,bfloat16,fp8,0,0.013334399461746216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,32,128,1,bfloat16,bfloat16,0,0.013129599392414093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,32,128,1,bfloat16,fp8,0,0.013227200508117676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,32,128,1,fp8,fp8,0,0.013407999277114868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,32,1,128,1,bfloat16,bfloat16,0,0.02910720109939575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,1,128,1,bfloat16,fp8,0,0.013276800513267517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,1,128,1,fp8,fp8,0,0.01327040046453476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,2,128,1,bfloat16,bfloat16,0,0.012787200510501862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,2,128,1,bfloat16,fp8,0,0.013254399597644805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,2,128,1,fp8,fp8,0,0.013324800133705138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,4,128,1,bfloat16,bfloat16,0,0.012614400684833526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,4,128,1,bfloat16,fp8,0,0.01334560066461563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,4,128,1,fp8,fp8,0,0.013227200508117676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,8,128,1,bfloat16,bfloat16,0,0.012720000743865967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,8,128,1,bfloat16,fp8,0,0.013406400382518769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,8,128,1,fp8,fp8,0,0.013544000685214996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,32,32,128,1,bfloat16,bfloat16,0,0.039684799313545224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,32,128,1,bfloat16,bfloat16,0,0.012751999497413635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,32,128,1,bfloat16,fp8,0,0.013096000254154205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,1,128,1,bfloat16,bfloat16,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,1,128,1,bfloat16,fp8,0,0.013072000443935394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,1,128,1,fp8,fp8,0,0.01297920048236847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,2,128,1,bfloat16,bfloat16,0,0.012414400279521943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,2,128,1,bfloat16,fp8,0,0.012961600720882416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,2,128,1,fp8,fp8,0,0.013070400059223174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,4,128,1,bfloat16,bfloat16,0,0.012348800152540206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,4,128,1,bfloat16,fp8,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,4,128,1,fp8,fp8,0,0.012848000228404998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,8,128,1,bfloat16,bfloat16,0,0.012510399520397186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,8,128,1,bfloat16,fp8,0,0.012918399274349212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,8,128,1,fp8,fp8,0,0.012806400656700134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,1,128,1,bfloat16,bfloat16,0,0.10298080444335937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,1,128,1,bfloat16,fp8,0,0.11059199571609497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,1,128,1,fp8,fp8,0,0.11050239801406861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,2,128,1,bfloat16,bfloat16,0,0.10609760284423828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,2,128,1,bfloat16,fp8,0,0.11208479404449463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,2,128,1,fp8,fp8,0,0.11121280193328857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,4,128,1,bfloat16,bfloat16,0,0.11104480028152466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,4,128,1,bfloat16,fp8,0,0.1144368052482605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,4,128,1,fp8,fp8,0,0.11416959762573242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,8,128,1,bfloat16,bfloat16,0,0.12295039892196655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,8,128,1,bfloat16,fp8,0,0.12023999691009521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,32,8,128,1,fp8,fp8,0,0.12168480157852173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,1,128,1,bfloat16,bfloat16,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,32,128,1,bfloat16,bfloat16,0,0.1883455991744995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,32,128,1,bfloat16,fp8,0,0.19275360107421874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,2,128,1,bfloat16,fp8,0,0.013422399759292603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,1,128,1,bfloat16,bfloat16,0,0.05889760255813599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,32,128,1,fp8,fp8,0,0.19513119459152223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,1,128,1,bfloat16,fp8,0,0.06153119802474975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,1,128,1,fp8,fp8,0,0.06092960238456726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,2,128,1,bfloat16,bfloat16,0,0.06041600108146668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,2,128,1,bfloat16,fp8,0,0.06254720091819763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,2,128,1,fp8,fp8,0,0.06229760050773621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,4,128,1,bfloat16,fp8,0,0.06365600228309631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,4,128,1,fp8,fp8,0,0.06338880062103272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,32,8,128,1,fp8,fp8,0,0.013291199505329133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,8,128,1,bfloat16,bfloat16,0,0.06880639791488648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,8,128,1,bfloat16,fp8,0,0.06595519781112671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,8,128,1,fp8,fp8,0,0.06608319878578187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,32,1,128,1,bfloat16,bfloat16,0,0.012300799787044524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,32,128,1,bfloat16,bfloat16,0,0.10167360305786133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,32,128,1,bfloat16,fp8,0,0.10159679651260375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,1,128,1,bfloat16,bfloat16,0,0.028968000411987306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,32,128,1,fp8,fp8,0,0.10136640071868896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,1,128,1,bfloat16,fp8,0,0.03020800054073334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,1,128,1,fp8,fp8,0,0.03052479922771454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,2,128,1,bfloat16,bfloat16,0,0.03160159885883331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,2,128,1,bfloat16,fp8,0,0.030403199791908263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,2,128,1,fp8,fp8,0,0.03028480112552643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,4,128,1,bfloat16,bfloat16,0,0.03558239936828613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,4,128,1,fp8,fp8,0,0.03138079941272735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,4,128,1,bfloat16,fp8,0,0.03131999969482422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,8,128,1,bfloat16,bfloat16,0,0.039575999975204466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,8,128,1,bfloat16,fp8,0,0.03306719958782196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,32,8,128,1,fp8,fp8,0,0.03214559853076935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,32,128,1,bfloat16,bfloat16,0,0.05522720217704773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,32,128,1,bfloat16,fp8,0,0.04945760071277618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,1,128,1,bfloat16,bfloat16,0,0.018142400681972502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,1,128,1,bfloat16,fp8,0,0.01960960030555725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,1,128,1,fp8,fp8,0,0.01958400011062622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,2,128,1,bfloat16,fp8,0,0.019356800615787505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,2,128,1,fp8,fp8,0,0.019500799477100372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,32,32,128,1,fp8,fp8,0,0.01334560066461563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,4,128,1,bfloat16,bfloat16,0,0.01871040016412735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,4,128,1,bfloat16,fp8,0,0.01950719952583313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,4,128,1,fp8,fp8,0,0.019574399292469024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,8,128,1,bfloat16,bfloat16,0,0.019289599359035493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,8,128,1,fp8,fp8,0,0.019670400023460387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,32,128,1,bfloat16,bfloat16,0,0.02627840042114258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,32,128,1,bfloat16,fp8,0,0.027806401252746582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,32,128,1,fp8,fp8,0,0.027833598852157592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,1,128,1,bfloat16,bfloat16,0,0.01316000074148178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,1,128,1,bfloat16,fp8,0,0.013913600146770478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,1,128,1,fp8,fp8,0,0.01369439959526062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,2,128,1,bfloat16,fp8,0,0.01390880048274994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,2,128,1,fp8,fp8,0,0.013787199556827546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,4,128,1,bfloat16,bfloat16,0,0.013441599905490875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,4,128,1,bfloat16,fp8,0,0.013886399567127228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,4,128,1,fp8,fp8,0,0.01398559957742691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,8,128,1,bfloat16,bfloat16,0,0.013579200208187103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,8,128,1,bfloat16,fp8,0,0.013980799913406372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,8,128,1,fp8,fp8,0,0.014017599821090698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,32,128,1,bfloat16,bfloat16,0,0.017641599476337432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,32,128,1,bfloat16,fp8,0,0.01820800006389618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,32,128,1,fp8,fp8,0,0.01814880073070526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,1,128,1,bfloat16,bfloat16,0,0.01290079951286316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,1,128,1,bfloat16,fp8,0,0.013529600203037262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,1,128,1,fp8,fp8,0,0.013657599687576294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,2,128,1,bfloat16,bfloat16,0,0.013041600584983826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,2,128,1,bfloat16,fp8,0,0.013575999438762665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,2,128,1,fp8,fp8,0,0.013579200208187103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,4,128,1,bfloat16,bfloat16,0,0.012958399951457977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,4,128,1,bfloat16,fp8,0,0.013687999546527862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,4,128,1,fp8,fp8,0,0.013462400436401368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,8,128,1,bfloat16,bfloat16,0,0.013388800621032714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,8,128,1,bfloat16,fp8,0,0.014131200313568116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,32,8,128,1,fp8,fp8,0,0.013606399297714233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,32,128,1,bfloat16,bfloat16,0,0.012996800243854523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,32,128,1,bfloat16,fp8,0,0.01350879967212677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,32,128,1,fp8,fp8,0,0.013484799861907959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,1,128,1,bfloat16,bfloat16,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,1,128,1,bfloat16,fp8,0,0.013035200536251068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,1,128,1,fp8,fp8,0,0.013140800595283508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,2,128,1,bfloat16,bfloat16,0,0.012467200309038163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,2,128,1,bfloat16,fp8,0,0.01313440054655075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,2,128,1,fp8,fp8,0,0.01313599944114685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,4,128,1,bfloat16,bfloat16,0,0.012438400089740754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,4,128,1,bfloat16,fp8,0,0.013145600259304047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,4,128,1,fp8,fp8,0,0.01313440054655075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,8,128,1,bfloat16,bfloat16,0,0.012705600261688233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,8,128,1,bfloat16,fp8,0,0.013209599256515502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,32,4,128,1,bfloat16,bfloat16,0,0.06326239705085754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,32,8,128,1,fp8,fp8,0,0.013353599607944489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,32,128,1,bfloat16,bfloat16,0,0.012828800082206725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,32,128,1,fp8,fp8,0,0.013059200346469879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,1,128,1,bfloat16,bfloat16,0,0.012247999757528305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,1,128,1,bfloat16,fp8,0,0.01271360069513321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,1,128,1,fp8,fp8,0,0.013123199343681335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,2,128,1,bfloat16,bfloat16,0,0.012283200025558471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,2,128,1,bfloat16,fp8,0,0.012913599610328674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,2,128,1,fp8,fp8,0,0.012649600207805634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,4,128,1,bfloat16,bfloat16,0,0.012377600371837615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,4,128,1,bfloat16,fp8,0,0.012851199507713318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,4,128,1,fp8,fp8,0,0.012899200618267059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,8,128,1,bfloat16,bfloat16,0,0.012388800084590913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,8,128,1,bfloat16,fp8,0,0.01268800050020218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,8,128,1,fp8,fp8,0,0.013073599338531494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,32,128,1,bfloat16,bfloat16,0,0.01250240057706833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,32,128,1,bfloat16,fp8,0,0.012992000579833985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,32,128,1,fp8,fp8,0,0.012902399897575379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,1,128,1,bfloat16,bfloat16,0,0.01210239976644516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,1,128,1,bfloat16,fp8,0,0.012691199779510498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,2,128,1,bfloat16,bfloat16,0,0.018980799615383147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,1,128,1,fp8,fp8,0,0.012622399628162384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,2,128,1,bfloat16,bfloat16,0,0.012054400146007537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,2,128,1,fp8,fp8,0,0.012614400684833526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,2,128,1,bfloat16,fp8,0,0.012943999469280243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,4,128,1,bfloat16,bfloat16,0,0.012216000258922577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,8,128,1,bfloat16,fp8,0,0.01993280053138733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,4,128,1,bfloat16,fp8,0,0.012643200159072877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,4,128,1,fp8,fp8,0,0.012923200428485871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,8,128,1,bfloat16,fp8,0,0.012856000661849975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,8,128,1,fp8,fp8,0,0.012601600587368011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,32,2,128,1,bfloat16,bfloat16,0,0.013169600069522858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,1,128,1,bfloat16,bfloat16,0,0.059875202178955075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,1,128,1,fp8,fp8,0,0.06230720281600952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,2,128,1,bfloat16,fp8,0,0.06330879926681518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,2,128,1,fp8,fp8,0,0.06312479972839355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,4,128,1,bfloat16,bfloat16,0,0.06453920006752015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,4,128,1,bfloat16,fp8,0,0.06441439986228943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,4,128,1,fp8,fp8,0,0.06434080004692078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,8,128,1,bfloat16,bfloat16,0,0.09909759759902954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,8,128,1,bfloat16,fp8,0,0.10201439857482911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,8,128,1,fp8,fp8,0,0.10196319818496705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,32,128,1,bfloat16,bfloat16,0,0.16007519960403443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,32,128,1,bfloat16,fp8,0,0.17664159536361695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,1,128,1,bfloat16,bfloat16,0,0.030689600110054015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,32,128,1,fp8,fp8,0,0.17575360536575318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,1,128,1,bfloat16,fp8,0,0.03163839876651764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,32,32,128,1,bfloat16,fp8,0,0.01329440027475357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,1,128,1,fp8,fp8,0,0.031700798869133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,2,128,1,bfloat16,bfloat16,0,0.03247359991073608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,2,128,1,bfloat16,fp8,0,0.03127999901771546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,2,128,1,fp8,fp8,0,0.03177599906921387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,4,128,1,bfloat16,bfloat16,0,0.036039999127388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,4,128,1,bfloat16,fp8,0,0.03159359991550446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,4,128,1,fp8,fp8,0,0.03219360113143921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,8,128,1,bfloat16,bfloat16,0,0.05608159899711609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,8,128,1,bfloat16,fp8,0,0.0508512020111084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,32,8,128,1,fp8,fp8,0,0.05228480100631714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,32,128,1,bfloat16,bfloat16,0,0.08611840009689331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,32,128,1,fp8,fp8,0,0.08636959791183471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,32,32,128,1,fp8,fp8,0,0.04881120026111603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,1,128,1,bfloat16,bfloat16,0,0.018984000384807586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,1,128,1,bfloat16,fp8,0,0.019972799718379973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,1,128,1,fp8,fp8,0,0.020294399559497835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,2,128,1,bfloat16,bfloat16,0,0.01903039962053299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,2,128,1,bfloat16,fp8,0,0.019947199523448943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,2,128,1,fp8,fp8,0,0.02034880071878433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,4,128,1,bfloat16,bfloat16,0,0.01942719966173172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,4,128,1,bfloat16,fp8,0,0.02011680006980896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,4,128,1,fp8,fp8,0,0.020390400290489198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,8,128,1,bfloat16,bfloat16,0,0.02648800015449524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,8,128,1,bfloat16,fp8,0,0.028401601314544677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,8,128,1,fp8,fp8,0,0.028951999545097352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,32,128,1,bfloat16,bfloat16,0,0.042033600807189944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,32,128,1,bfloat16,fp8,0,0.046024000644683837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,1,128,1,bfloat16,bfloat16,0,0.01406240016222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,1,128,1,bfloat16,fp8,0,0.014238399267196656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,1,128,1,fp8,fp8,0,0.0146479994058609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,2,128,1,bfloat16,bfloat16,0,0.01361439973115921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,2,128,1,bfloat16,bfloat16,0,0.06077439785003662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,2,128,1,bfloat16,fp8,0,0.014603200554847717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,2,128,1,fp8,fp8,0,0.01440960019826889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,4,128,1,bfloat16,fp8,0,0.014686399698257446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,4,128,1,fp8,fp8,0,0.014427199959754944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,8,128,1,bfloat16,bfloat16,0,0.01751520037651062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,8,128,1,bfloat16,fp8,0,0.01836639940738678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,8,128,1,fp8,fp8,0,0.018638400733470915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,32,128,1,bfloat16,bfloat16,0,0.025164800882339477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,32,128,1,bfloat16,fp8,0,0.02720000147819519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,32,128,1,fp8,fp8,0,0.0270224004983902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,1,128,1,bfloat16,bfloat16,0,0.012771199643611907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,1,128,1,bfloat16,fp8,0,0.013697600364685059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,1,128,1,fp8,fp8,0,0.013726399838924408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,2,128,1,bfloat16,bfloat16,0,0.013041600584983826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,2,128,1,bfloat16,fp8,0,0.013804799318313599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,4,128,1,bfloat16,bfloat16,0,0.012940800189971924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,4,128,1,bfloat16,fp8,0,0.01356319934129715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,4,128,1,fp8,fp8,0,0.013820800185203552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,8,128,1,bfloat16,bfloat16,0,0.012859199941158295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,8,128,1,bfloat16,fp8,0,0.013489599525928497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,8,128,1,fp8,fp8,0,0.013544000685214996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,32,128,1,bfloat16,bfloat16,0,0.016990399360656737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,32,128,1,bfloat16,fp8,0,0.018265600502490997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,32,128,1,fp8,fp8,0,0.017827199399471284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,1,128,1,bfloat16,bfloat16,0,0.012870399653911591
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,1,128,1,bfloat16,fp8,0,0.013364799320697784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,1,128,1,fp8,fp8,0,0.013359999656677246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,2,128,1,bfloat16,bfloat16,0,0.012780800461769104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,2,128,1,bfloat16,fp8,0,0.013352000713348388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,2,128,1,fp8,fp8,0,0.013335999846458436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,4,128,1,bfloat16,bfloat16,0,0.013105599582195282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,4,128,1,bfloat16,fp8,0,0.013126400113105775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,4,128,1,fp8,fp8,0,0.0135903999209404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,8,128,1,bfloat16,bfloat16,0,0.012481600046157837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,8,128,1,bfloat16,fp8,0,0.013459199666976928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,32,8,128,1,fp8,fp8,0,0.013072000443935394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,32,128,1,bfloat16,bfloat16,0,0.013236799836158752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,32,128,1,bfloat16,fp8,0,0.01318880021572113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,32,128,1,fp8,fp8,0,0.013596799969673157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,1,128,1,bfloat16,bfloat16,0,0.012191999703645706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,1,128,1,bfloat16,fp8,0,0.0134320005774498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,1,128,1,fp8,fp8,0,0.012788799405097962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,2,128,1,bfloat16,bfloat16,0,0.012510399520397186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,2,128,1,bfloat16,fp8,0,0.01284320056438446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,2,128,1,fp8,fp8,0,0.012988799810409546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,4,128,1,bfloat16,bfloat16,0,0.012191999703645706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,4,128,1,bfloat16,fp8,0,0.012993599474430084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,4,128,1,fp8,fp8,0,0.012856000661849975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,32,32,128,1,bfloat16,fp8,0,0.08670719861984252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,8,128,1,bfloat16,bfloat16,0,0.012241599708795547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,8,128,1,bfloat16,fp8,0,0.012865599989891053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,32,8,128,1,fp8,fp8,0,0.012950399518013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,32,128,1,bfloat16,bfloat16,0,0.012614400684833526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,32,128,1,bfloat16,fp8,0,0.013236799836158752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,32,128,1,fp8,fp8,0,0.013019199669361114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,1,128,1,bfloat16,bfloat16,0,0.01231200024485588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,1,128,1,bfloat16,fp8,0,0.012668800354003907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,1,128,1,fp8,fp8,0,0.012955200672149659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,2,128,1,bfloat16,bfloat16,0,0.012188799679279327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,32,8,128,1,bfloat16,bfloat16,0,0.012203200161457062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,2,128,1,bfloat16,fp8,0,0.012931199371814727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,2,128,1,fp8,fp8,0,0.01284639984369278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,4,128,1,bfloat16,bfloat16,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,4,128,1,bfloat16,fp8,0,0.01297439932823181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,4,128,1,fp8,fp8,0,0.012718400359153748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,8,128,1,bfloat16,bfloat16,0,0.011881600320339202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,8,128,1,bfloat16,fp8,0,0.012774400413036346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,32,1,128,1,bfloat16,fp8,0,0.06258559823036194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,32,8,128,1,fp8,fp8,0,0.012520000338554382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,32,128,1,bfloat16,bfloat16,0,0.012511999905109405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,1,128,1,bfloat16,bfloat16,0,0.011903999745845795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,32,128,1,fp8,fp8,0,0.012908799946308136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,1,128,1,bfloat16,fp8,0,0.01255200058221817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,1,128,1,fp8,fp8,0,0.012591999769210816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,2,128,1,bfloat16,fp8,0,0.012516799569129943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,4,128,1,bfloat16,bfloat16,0,0.014095999300479889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,4,128,1,bfloat16,bfloat16,0,0.011881600320339202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,4,128,1,bfloat16,fp8,0,0.01263359934091568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,4,128,1,fp8,fp8,0,0.012518399953842163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,8,128,1,bfloat16,bfloat16,0,0.011777599900960922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,8,128,1,bfloat16,fp8,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,8,128,1,fp8,fp8,0,0.012529599666595458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,1,128,1,bfloat16,bfloat16,0,0.021279999613761903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,1,128,1,bfloat16,fp8,0,0.02308479994535446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,1,128,1,fp8,fp8,0,0.023129600286483764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,2,128,1,bfloat16,bfloat16,0,0.02847839891910553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,2,128,1,bfloat16,fp8,0,0.031611201167106626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,32,2,128,1,fp8,fp8,0,0.013736000657081604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,2,128,1,fp8,fp8,0,0.03186880052089691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,4,128,1,bfloat16,bfloat16,0,0.04290400147438049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,4,128,1,bfloat16,fp8,0,0.049025601148605345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,4,128,1,fp8,fp8,0,0.049316799640655516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,8,128,1,bfloat16,bfloat16,0,0.07084479928016663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,8,128,1,bfloat16,fp8,0,0.08425120115280152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,32,8,128,1,fp8,fp8,0,0.0847487986087799
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,32,128,1,bfloat16,bfloat16,0,0.1255887985229492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,1,128,1,bfloat16,bfloat16,0,0.015083199739456177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,32,128,1,bfloat16,fp8,0,0.15497599840164183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,32,128,1,fp8,fp8,0,0.15499039888381957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,1,128,1,bfloat16,fp8,0,0.015806399285793304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,1,128,1,fp8,fp8,0,0.01597920060157776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,2,128,1,bfloat16,bfloat16,0,0.018243199586868285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,2,128,1,bfloat16,fp8,0,0.020212799310684204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,2,128,1,fp8,fp8,0,0.020139199495315552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,4,128,1,bfloat16,bfloat16,0,0.02573919892311096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,4,128,1,bfloat16,fp8,0,0.028990399837493897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,4,128,1,fp8,fp8,0,0.02922079861164093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,8,128,1,bfloat16,bfloat16,0,0.04010879993438721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,8,128,1,bfloat16,fp8,0,0.04641759991645813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,32,8,128,1,fp8,fp8,0,0.04669919908046723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,32,128,1,bfloat16,bfloat16,0,0.06914560198783874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,32,128,1,bfloat16,fp8,0,0.08158400058746337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,32,128,1,fp8,fp8,0,0.08216800093650818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,1,128,1,bfloat16,bfloat16,0,0.013167999684810638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,1,128,1,bfloat16,fp8,0,0.014324800670146942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,1,128,1,fp8,fp8,0,0.014428800344467163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,2,128,1,bfloat16,bfloat16,0,0.013273599743843078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,2,128,1,bfloat16,fp8,0,0.014425599575042724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,2,128,1,fp8,fp8,0,0.014459200203418732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,4,128,1,bfloat16,bfloat16,0,0.017179200053215028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,4,128,1,bfloat16,fp8,0,0.018721599876880646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,4,128,1,fp8,fp8,0,0.018911999464035035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,8,128,1,bfloat16,bfloat16,0,0.024454399943351746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,8,128,1,bfloat16,fp8,0,0.027505600452423097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,32,8,128,1,fp8,fp8,0,0.027657601237297057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,32,128,1,bfloat16,bfloat16,0,0.03967680037021637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,32,128,1,bfloat16,fp8,0,0.04502719938755036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,32,128,1,fp8,fp8,0,0.04499199986457825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,1,128,1,bfloat16,fp8,0,0.013388800621032714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,1,128,1,fp8,fp8,0,0.013219200074672699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,2,128,1,bfloat16,bfloat16,0,0.012443199753761292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,2,128,1,bfloat16,fp8,0,0.013246400654315949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,2,128,1,fp8,fp8,0,0.013380800187587739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,4,128,1,bfloat16,bfloat16,0,0.01260959953069687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,4,128,1,bfloat16,fp8,0,0.013451200723648072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,4,128,1,fp8,fp8,0,0.013451200723648072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,8,128,1,bfloat16,bfloat16,0,0.016339200735092162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,8,128,1,bfloat16,fp8,0,0.017871999740600587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,8,128,1,fp8,fp8,0,0.017827199399471284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,32,128,1,bfloat16,bfloat16,0,0.02402399927377701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,32,128,1,bfloat16,fp8,0,0.026521599292755126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,32,128,1,fp8,fp8,0,0.026529601216316222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,32,32,128,1,fp8,fp8,0,0.046961599588394166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,1,128,1,bfloat16,fp8,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,1,128,1,fp8,fp8,0,0.012777599692344665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,2,128,1,bfloat16,bfloat16,0,0.011865600198507308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,2,128,1,bfloat16,fp8,0,0.012856000661849975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,2,128,1,fp8,fp8,0,0.012646399438381195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,4,128,1,bfloat16,bfloat16,0,0.012153600156307221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,2,128,1,fp8,fp8,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,4,128,1,bfloat16,fp8,0,0.012681600451469422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,4,128,1,fp8,fp8,0,0.013012799620628356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,8,128,1,bfloat16,bfloat16,0,0.012019199877977371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,8,128,1,bfloat16,fp8,0,0.013100799918174744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,8,128,1,fp8,fp8,0,0.012835200130939483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,32,128,1,bfloat16,bfloat16,0,0.016599999368190767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,32,128,1,bfloat16,fp8,0,0.017441600561141968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,1,128,1,bfloat16,bfloat16,0,0.011793600022792816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,1,128,1,bfloat16,fp8,0,0.01247360035777092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,1,128,1,fp8,fp8,0,0.01249760016798973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,2,128,1,bfloat16,bfloat16,0,0.011840000003576278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,2,128,1,bfloat16,fp8,0,0.012563200294971466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,2,128,1,fp8,fp8,0,0.012593600153923034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,4,128,1,bfloat16,bfloat16,0,0.011843200027942657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,4,128,1,bfloat16,fp8,0,0.012678399682044983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,4,128,1,fp8,fp8,0,0.012649600207805634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,8,128,1,bfloat16,bfloat16,0,0.012033600360155106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,8,128,1,bfloat16,fp8,0,0.012731200456619263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,8,128,1,fp8,fp8,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,32,128,1,bfloat16,bfloat16,0,0.012558400630950928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,32,128,1,bfloat16,fp8,0,0.013016000390052795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,32,128,1,fp8,fp8,0,0.01297599971294403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,1,128,1,bfloat16,bfloat16,0,0.011401599645614624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,1,128,1,fp8,fp8,0,0.012148799747228623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,2,128,1,bfloat16,bfloat16,0,0.01146719977259636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,2,128,1,bfloat16,fp8,0,0.012139199674129486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,2,128,1,fp8,fp8,0,0.012201599776744843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,4,128,1,bfloat16,bfloat16,0,0.011763200163841248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,32,1,128,1,bfloat16,bfloat16,0,0.012707200646400452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,4,128,1,fp8,fp8,0,0.012372799962759019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,8,128,1,bfloat16,bfloat16,0,0.011691199988126755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,8,128,1,bfloat16,fp8,0,0.012491200119256973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,8,128,1,fp8,fp8,0,0.01263359934091568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,32,128,1,bfloat16,bfloat16,0,0.012518399953842163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,32,128,1,bfloat16,fp8,0,0.012848000228404998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,32,128,1,fp8,fp8,0,0.012822400033473968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,32,128,1,bfloat16,fp8,0,0.013079999387264252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,1,128,1,bfloat16,bfloat16,0,0.011448000371456147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,1,128,1,bfloat16,fp8,0,0.012368000298738479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,32,2,128,1,bfloat16,bfloat16,0,0.011823999881744384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,2,128,1,bfloat16,bfloat16,0,0.011686400324106217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,2,128,1,bfloat16,fp8,0,0.01215519979596138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,2,128,1,fp8,fp8,0,0.01234079971909523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,4,128,1,bfloat16,bfloat16,0,0.011473599821329117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,4,128,1,bfloat16,fp8,0,0.012336000055074691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,4,128,1,fp8,fp8,0,0.012118399888277055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,8,128,1,bfloat16,bfloat16,0,0.011723200231790543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,8,128,1,bfloat16,fp8,0,0.012462399899959564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,8,128,1,fp8,fp8,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,32,32,128,1,fp8,fp8,0,0.017580799758434296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,32,128,1,bfloat16,bfloat16,0,0.012299200147390365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,32,128,1,fp8,fp8,0,0.013067199289798737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,1,128,1,bfloat16,fp8,0,0.012375999987125397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,1,128,1,fp8,fp8,0,0.012228800356388092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,2,128,1,bfloat16,bfloat16,0,0.011561600118875503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,2,128,1,bfloat16,fp8,0,0.012243200093507767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,2,128,1,fp8,fp8,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,4,128,1,bfloat16,bfloat16,0,0.01156959980726242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,4,128,1,bfloat16,fp8,0,0.01236959993839264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,4,128,1,fp8,fp8,0,0.012358400225639343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,8,128,1,bfloat16,bfloat16,0,0.011707200109958649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,8,128,1,bfloat16,fp8,0,0.012459199875593185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,8,128,1,fp8,fp8,0,0.01244800016283989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,1,128,1,bfloat16,fp8,0,0.012385600060224534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,32,4,128,1,bfloat16,fp8,0,0.012571200728416443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,32,1,128,1,bfloat16,bfloat16,0,0.01186719983816147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,32,1,128,1,fp8,fp8,0,0.0121568001806736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,32,128,1,bfloat16,fp8,0,0.013022400438785553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,32,1,128,1,bfloat16,bfloat16,0,0.011550399661064147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,1,128,1,bfloat16,fp8,0,18.586302185058592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,1,128,1,fp8,fp8,0,18.397105407714843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,2,128,1,bfloat16,fp8,0,18.61139373779297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,2,128,1,fp8,fp8,0,18.70767822265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,1,128,1,bfloat16,bfloat16,0,22.248489379882812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,2,128,1,bfloat16,bfloat16,0,21.916545104980468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,4,128,1,bfloat16,bfloat16,0,22.142253112792968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,4,128,1,bfloat16,fp8,0,19.023199462890624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,24,128,1,bfloat16,bfloat16,0,12.10297088623047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,1,128,1,bfloat16,bfloat16,0,11.323883056640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,4,128,1,fp8,fp8,0,20.053782653808593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,8,128,1,bfloat16,fp8,0,20.451300048828124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,8,128,1,fp8,fp8,0,20.516905212402342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,24,8,128,1,bfloat16,bfloat16,0,24.21204528808594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,1,128,1,bfloat16,fp8,0,9.833190155029296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,1,128,1,fp8,fp8,0,9.504576110839844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,2,128,1,bfloat16,bfloat16,0,11.52929916381836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,2,128,1,bfloat16,fp8,0,9.46242904663086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,2,128,1,fp8,fp8,0,9.708455657958984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,4,128,1,bfloat16,fp8,0,9.421820831298827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,4,128,1,bfloat16,bfloat16,0,11.14884033203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,4,128,1,fp8,fp8,0,9.531903839111328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,24,128,1,bfloat16,bfloat16,0,5.824910354614258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,8,128,1,bfloat16,fp8,0,9.452236938476563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,8,128,1,fp8,fp8,0,9.688807678222656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,8,128,1,bfloat16,bfloat16,0,11.27643051147461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,1,128,1,bfloat16,fp8,0,4.620713424682617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,1,128,1,bfloat16,bfloat16,0,5.796211242675781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,1,128,1,fp8,fp8,0,4.9235790252685545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,2,128,1,bfloat16,bfloat16,0,5.561361694335938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,2,128,1,bfloat16,fp8,0,4.863623809814453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,2,128,1,fp8,fp8,0,4.8804161071777346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,4,128,1,bfloat16,bfloat16,0,6.2600959777832035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,4,128,1,bfloat16,fp8,0,5.212884902954102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,4,128,1,fp8,fp8,0,5.137887954711914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,8,128,1,bfloat16,bfloat16,0,5.847715377807617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,24,128,1,bfloat16,bfloat16,0,2.992572784423828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,8,128,1,bfloat16,fp8,0,4.753595352172852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,8,128,1,fp8,fp8,0,5.143268966674805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,1,128,1,bfloat16,bfloat16,0,2.6354080200195313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,1,128,1,bfloat16,fp8,0,2.395427131652832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,1,128,1,fp8,fp8,0,2.5415279388427736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,2,128,1,bfloat16,bfloat16,0,2.7515632629394533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,2,128,1,bfloat16,fp8,0,2.3990432739257814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,2,128,1,fp8,fp8,0,2.5357711791992186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,4,128,1,bfloat16,bfloat16,0,2.552902412414551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,4,128,1,bfloat16,fp8,0,2.352779197692871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,4,128,1,fp8,fp8,0,2.6169599533081054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,8,128,1,bfloat16,bfloat16,0,2.7723791122436525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,8,128,1,bfloat16,fp8,0,2.40655517578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,8,128,1,fp8,fp8,0,2.467518424987793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,1,128,1,bfloat16,fp8,0,10.639390563964843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,1,128,1,bfloat16,bfloat16,0,12.793806457519532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,1,128,1,fp8,fp8,0,11.450180816650391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,2,128,1,bfloat16,bfloat16,0,13.467338562011719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,2,128,1,bfloat16,fp8,0,11.028520202636718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,2,128,1,fp8,fp8,0,10.799816131591797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,4,128,1,bfloat16,bfloat16,0,13.478994750976563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,4,128,1,bfloat16,fp8,0,10.903404998779298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,4,128,1,fp8,fp8,0,11.198478698730469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,8,128,1,bfloat16,bfloat16,0,14.18603057861328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,8,128,1,bfloat16,fp8,0,11.178833770751954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,24,8,128,1,fp8,fp8,0,11.042787170410156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,24,128,1,bfloat16,bfloat16,0,6.97701416015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,24,128,1,bfloat16,fp8,0,2.427475166320801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,24,24,128,1,fp8,fp8,0,2.420627212524414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,24,128,1,fp8,fp8,0,4.84779052734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,24,24,128,1,bfloat16,fp8,0,4.808254241943359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,24,128,1,bfloat16,fp8,0,5.563905715942383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,1,128,1,bfloat16,fp8,0,5.330310440063476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,24,128,1,fp8,fp8,0,5.556710433959961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,1,128,1,bfloat16,bfloat16,0,6.399863815307617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,24,128,1,bfloat16,fp8,0,9.607158660888672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,1,128,1,fp8,fp8,0,5.333193588256836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,2,128,1,bfloat16,bfloat16,0,6.323635101318359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,2,128,1,fp8,fp8,0,5.3091888427734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,4,128,1,bfloat16,fp8,0,5.369168090820312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,4,128,1,fp8,fp8,0,5.3502849578857425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,4,128,1,bfloat16,bfloat16,0,6.471598052978516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,24,24,128,1,fp8,fp8,0,9.614169311523437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,2,128,1,bfloat16,fp8,0,5.313148880004883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,24,128,1,bfloat16,fp8,0,2.7695600509643556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,8,128,1,bfloat16,bfloat16,0,6.678244781494141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,24,128,1,bfloat16,bfloat16,0,3.3188735961914064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,24,128,1,fp8,fp8,0,2.9588592529296873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,8,128,1,bfloat16,fp8,0,5.379428863525391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,24,8,128,1,fp8,fp8,0,5.356179046630859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,1,128,1,bfloat16,bfloat16,0,3.081563186645508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,1,128,1,bfloat16,fp8,0,2.627854347229004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,1,128,1,fp8,fp8,0,2.6373712539672853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,2,128,1,fp8,fp8,0,2.649550437927246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,2,128,1,bfloat16,fp8,0,2.9680063247680666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,2,128,1,bfloat16,bfloat16,0,3.14354248046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,4,128,1,bfloat16,fp8,0,2.667953681945801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,4,128,1,bfloat16,bfloat16,0,3.2177696228027344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,4,128,1,fp8,fp8,0,2.755183982849121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,24,128,1,bfloat16,fp8,0,1.534233570098877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,24,128,1,bfloat16,bfloat16,0,1.6467071533203126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,8,128,1,bfloat16,fp8,0,2.6764032363891603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,8,128,1,bfloat16,bfloat16,0,3.0728416442871094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,24,128,1,fp8,fp8,0,1.4014304161071778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,24,8,128,1,fp8,fp8,0,2.684294319152832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,1,128,1,bfloat16,bfloat16,0,1.583351993560791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,1,128,1,bfloat16,fp8,0,1.3200079917907714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,1,128,1,fp8,fp8,0,1.3393744468688964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,2,128,1,bfloat16,bfloat16,0,1.5338047981262206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,2,128,1,bfloat16,fp8,0,1.3264575958251954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,2,128,1,fp8,fp8,0,1.556345558166504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,4,128,1,bfloat16,bfloat16,0,1.505577564239502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,4,128,1,bfloat16,fp8,0,1.345628833770752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,4,128,1,fp8,fp8,0,1.4064111709594727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,8,128,1,bfloat16,bfloat16,0,1.5387264251708985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,8,128,1,bfloat16,fp8,0,1.349409580230713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,24,8,128,1,fp8,fp8,0,1.5572511672973632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,1,128,1,bfloat16,fp8,0,7.44140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,1,128,1,fp8,fp8,0,7.571403503417969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,2,128,1,bfloat16,fp8,0,7.417849731445313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,2,128,1,fp8,fp8,0,7.472921752929688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,1,128,1,bfloat16,bfloat16,0,9.410530853271485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,4,128,1,bfloat16,fp8,0,7.549531555175781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,2,128,1,bfloat16,bfloat16,0,9.089663696289062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,4,128,1,bfloat16,bfloat16,0,9.781130981445312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,24,128,1,bfloat16,fp8,0,4.311088180541992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,24,128,1,fp8,fp8,0,4.566993713378906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,24,128,1,bfloat16,bfloat16,0,5.428043365478516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,4,128,1,fp8,fp8,0,7.55994873046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,1,128,1,bfloat16,bfloat16,0,5.069723129272461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,8,128,1,bfloat16,fp8,0,8.445333099365234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,8,128,1,fp8,fp8,0,7.993724822998047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,1,128,1,bfloat16,fp8,0,3.813027191162109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,1,128,1,fp8,fp8,0,3.907729721069336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,24,8,128,1,bfloat16,bfloat16,0,10.263272094726563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,2,128,1,bfloat16,fp8,0,3.824214553833008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,2,128,1,bfloat16,bfloat16,0,4.4759777069091795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,2,128,1,fp8,fp8,0,3.9651119232177736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,4,128,1,bfloat16,bfloat16,0,4.414710235595703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,4,128,1,bfloat16,fp8,0,4.157788848876953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,4,128,1,fp8,fp8,0,3.7686046600341796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,24,128,1,bfloat16,bfloat16,0,2.43843994140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,8,128,1,bfloat16,bfloat16,0,4.542881774902344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,24,128,1,fp8,fp8,0,1.9854944229125977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,24,128,1,bfloat16,fp8,0,2.1013296127319334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,8,128,1,bfloat16,fp8,0,3.8083263397216798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,24,8,128,1,fp8,fp8,0,3.88653450012207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,1,128,1,bfloat16,bfloat16,0,2.074007987976074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,1,128,1,bfloat16,fp8,0,2.0987760543823244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,1,128,1,fp8,fp8,0,1.986123275756836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,2,128,1,bfloat16,fp8,0,1.9074304580688477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,2,128,1,fp8,fp8,0,1.8928672790527343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,2,128,1,bfloat16,bfloat16,0,2.161476707458496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,4,128,1,bfloat16,bfloat16,0,2.0901824951171877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,4,128,1,bfloat16,fp8,0,2.2639984130859374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,24,128,1,bfloat16,fp8,0,1.1436528205871581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,24,128,1,bfloat16,bfloat16,0,1.2281599998474122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,4,128,1,fp8,fp8,0,1.8791936874389648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,24,128,1,fp8,fp8,0,1.0876543998718262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,8,128,1,bfloat16,bfloat16,0,2.2404495239257813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,8,128,1,bfloat16,fp8,0,1.8956687927246094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,24,8,128,1,fp8,fp8,0,1.888435173034668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,1,128,1,bfloat16,bfloat16,0,1.0546815872192383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,1,128,1,bfloat16,fp8,0,0.9794912338256836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,2,128,1,bfloat16,bfloat16,0,1.0180512428283692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,1,128,1,fp8,fp8,0,1.1573951721191407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,2,128,1,bfloat16,fp8,0,1.0204832077026367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,4,128,1,bfloat16,fp8,0,0.9422112464904785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,2,128,1,fp8,fp8,0,1.1225760459899903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,4,128,1,fp8,fp8,0,1.0572992324829102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,8,128,1,bfloat16,bfloat16,0,1.0423727989196778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,8,128,1,bfloat16,fp8,0,1.030684757232666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,8,128,1,fp8,fp8,0,1.1246368408203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,24,4,128,1,bfloat16,bfloat16,0,1.0292624473571776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,1,128,1,bfloat16,fp8,0,9.741577911376954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,1,128,1,fp8,fp8,0,9.821115112304687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,2,128,1,bfloat16,fp8,0,9.773451232910157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,2,128,1,fp8,fp8,0,9.833137512207031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,1,128,1,bfloat16,bfloat16,0,11.251699066162109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,2,128,1,bfloat16,bfloat16,0,11.723990631103515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,4,128,1,bfloat16,bfloat16,0,12.32632827758789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,4,128,1,bfloat16,fp8,0,9.855174255371093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,24,128,1,bfloat16,bfloat16,0,6.403598022460938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,24,128,1,bfloat16,fp8,0,5.259403228759766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,24,128,1,fp8,fp8,0,5.197633743286133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,1,128,1,bfloat16,bfloat16,0,5.836558532714844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,4,128,1,fp8,fp8,0,9.934088134765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,8,128,1,bfloat16,fp8,0,9.88719482421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,8,128,1,fp8,fp8,0,9.992755126953124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,24,8,128,1,bfloat16,bfloat16,0,12.633971405029296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,1,128,1,fp8,fp8,0,5.189054489135742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,1,128,1,bfloat16,fp8,0,5.22542724609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,2,128,1,bfloat16,bfloat16,0,5.805673599243164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,2,128,1,bfloat16,fp8,0,4.864033508300781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,2,128,1,fp8,fp8,0,5.053420639038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,4,128,1,bfloat16,fp8,0,4.95996322631836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,4,128,1,bfloat16,bfloat16,0,6.638568115234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,24,128,1,bfloat16,fp8,0,2.6071552276611327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,24,128,1,bfloat16,bfloat16,0,3.0329168319702147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,4,128,1,fp8,fp8,0,5.323160171508789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,8,128,1,bfloat16,fp8,0,5.165035247802734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,24,128,1,fp8,fp8,0,2.6830560684204103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,8,128,1,bfloat16,bfloat16,0,6.131558227539062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,24,8,128,1,fp8,fp8,0,5.00731201171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,1,128,1,bfloat16,bfloat16,0,2.9502384185791017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,1,128,1,bfloat16,fp8,0,2.459985542297363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,1,128,1,fp8,fp8,0,2.6222015380859376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,2,128,1,bfloat16,fp8,0,2.441187286376953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,2,128,1,bfloat16,bfloat16,0,2.926355171203613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,2,128,1,fp8,fp8,0,2.8323360443115235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,4,128,1,bfloat16,fp8,0,2.4489343643188475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,4,128,1,bfloat16,bfloat16,0,3.056656074523926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,24,128,1,bfloat16,bfloat16,0,1.7413631439208985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,4,128,1,fp8,fp8,0,2.6342111587524415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,8,128,1,bfloat16,bfloat16,0,2.9316432952880858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,24,128,1,bfloat16,fp8,0,1.5242688179016113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,8,128,1,fp8,fp8,0,2.5200576782226562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,24,128,1,fp8,fp8,0,1.3334447860717773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,1,128,1,bfloat16,bfloat16,0,1.5255087852478026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,1,128,1,fp8,fp8,0,1.2191503524780274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,1,128,1,bfloat16,fp8,0,1.437622356414795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,2,128,1,bfloat16,fp8,0,1.3594271659851074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,2,128,1,bfloat16,bfloat16,0,1.478713607788086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,2,128,1,fp8,fp8,0,1.2290911674499512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,4,128,1,bfloat16,bfloat16,0,1.3937024116516112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,4,128,1,bfloat16,fp8,0,1.3970447540283204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,4,128,1,fp8,fp8,0,1.232703971862793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,24,128,1,bfloat16,bfloat16,0,0.7354623794555664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,8,128,1,bfloat16,bfloat16,0,1.4025504112243652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,24,128,1,bfloat16,fp8,0,0.7733888149261474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,24,128,1,fp8,fp8,0,0.6779488086700439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,8,128,1,bfloat16,fp8,0,1.3366000175476074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,1,128,1,bfloat16,bfloat16,0,0.6845759868621826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,24,8,128,1,fp8,fp8,0,1.4715807914733887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,1,128,1,bfloat16,fp8,0,0.6301536083221435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,1,128,1,fp8,fp8,0,0.7227968215942383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,2,128,1,bfloat16,bfloat16,0,0.7209311962127686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,2,128,1,bfloat16,fp8,0,0.6264880180358887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,2,128,1,fp8,fp8,0,0.671780776977539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,24,8,128,1,bfloat16,fp8,0,2.4837215423583983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,4,128,1,bfloat16,fp8,0,0.6545455932617188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,4,128,1,bfloat16,bfloat16,0,0.7008111953735352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,4,128,1,fp8,fp8,0,0.7054671764373779
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,8,128,1,bfloat16,fp8,0,0.6274784088134766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,8,128,1,bfloat16,bfloat16,0,0.735423994064331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,24,8,128,1,fp8,fp8,0,0.6739295959472656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,1,128,1,bfloat16,fp8,0,5.6256561279296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,2,128,1,bfloat16,fp8,0,5.6220703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,1,128,1,bfloat16,bfloat16,0,6.758238220214844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,2,128,1,fp8,fp8,0,5.917916870117187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,2,128,1,bfloat16,bfloat16,0,6.827490997314453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,4,128,1,bfloat16,bfloat16,0,7.2311042785644535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,1,128,1,fp8,fp8,0,5.587519836425781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,24,128,1,bfloat16,bfloat16,0,3.7092208862304688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,24,128,1,bfloat16,fp8,0,3.0906463623046876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,4,128,1,bfloat16,fp8,0,5.669779205322266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,4,128,1,fp8,fp8,0,5.7373615264892575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,8,128,1,bfloat16,fp8,0,5.772568130493164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,24,128,1,fp8,fp8,0,3.027332878112793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,8,128,1,fp8,fp8,0,6.104665756225586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,24,8,128,1,bfloat16,bfloat16,0,6.870489501953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,1,128,1,bfloat16,fp8,0,2.802836799621582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,1,128,1,bfloat16,bfloat16,0,3.2137630462646483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,1,128,1,fp8,fp8,0,2.793441581726074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,2,128,1,bfloat16,bfloat16,0,3.2998687744140627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,2,128,1,bfloat16,fp8,0,2.8079423904418945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,2,128,1,fp8,fp8,0,2.8081296920776366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,4,128,1,bfloat16,bfloat16,0,3.3990463256835937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,4,128,1,bfloat16,fp8,0,2.8706735610961913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,4,128,1,fp8,fp8,0,2.9735584259033203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,24,128,1,bfloat16,fp8,0,1.5198800086975097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,24,128,1,bfloat16,bfloat16,0,1.7808223724365235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,8,128,1,bfloat16,fp8,0,2.938572883605957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,8,128,1,bfloat16,bfloat16,0,3.5124126434326173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,24,128,1,fp8,fp8,0,1.7869695663452148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,1,128,1,bfloat16,fp8,0,1.409596824645996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,24,8,128,1,fp8,fp8,0,3.2596176147460936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,1,128,1,fp8,fp8,0,1.386899185180664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,2,128,1,bfloat16,fp8,0,1.4882384300231934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,2,128,1,bfloat16,bfloat16,0,1.7026592254638673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,2,128,1,fp8,fp8,0,1.6369007110595704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,4,128,1,bfloat16,bfloat16,0,1.5946831703186035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,4,128,1,bfloat16,fp8,0,1.534545612335205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,4,128,1,fp8,fp8,0,1.4388447761535645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,8,128,1,bfloat16,bfloat16,0,1.5731231689453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,8,128,1,bfloat16,fp8,0,1.5041760444641112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,24,128,1,bfloat16,bfloat16,0,0.9263680458068848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,24,128,1,bfloat16,fp8,0,0.8150447845458985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,8,128,1,fp8,fp8,0,1.574294376373291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,24,128,1,fp8,fp8,0,0.7795695781707763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,1,128,1,bfloat16,bfloat16,0,0.7521632194519043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,1,128,1,bfloat16,fp8,0,0.831657600402832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,1,128,1,fp8,fp8,0,0.738966417312622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,2,128,1,bfloat16,fp8,0,0.7369135856628418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,2,128,1,fp8,fp8,0,0.710262393951416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,2,128,1,bfloat16,bfloat16,0,0.8705056190490723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,4,128,1,bfloat16,bfloat16,0,0.817204761505127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,4,128,1,bfloat16,fp8,0,0.7174367904663086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,4,128,1,fp8,fp8,0,0.7938208103179931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,8,128,1,bfloat16,bfloat16,0,0.8002927780151368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,8,128,1,bfloat16,fp8,0,0.727894401550293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,24,128,1,bfloat16,fp8,0,0.4348735809326172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,24,8,128,1,fp8,fp8,0,0.7250512123107911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,1,128,1,bfloat16,bfloat16,0,0.399782395362854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,1,128,1,bfloat16,fp8,0,0.3824656009674072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,1,128,1,fp8,fp8,0,0.3579807996749878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,2,128,1,bfloat16,fp8,0,0.35869441032409666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,2,128,1,bfloat16,bfloat16,0,0.41503357887268066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,2,128,1,fp8,fp8,0,0.37694079875946046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,4,128,1,bfloat16,bfloat16,0,0.39991519451141355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,4,128,1,bfloat16,fp8,0,0.36997280120849607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,4,128,1,fp8,fp8,0,0.3625231981277466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,8,128,1,bfloat16,bfloat16,0,0.4150911808013916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,8,128,1,bfloat16,fp8,0,0.38412320613861084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,8,128,1,fp8,fp8,0,0.3863584041595459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,24,1,128,1,bfloat16,bfloat16,0,1.589022445678711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,24,128,1,bfloat16,bfloat16,0,0.5330575942993164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,24,24,128,1,fp8,fp8,0,0.39623360633850097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,1,128,1,bfloat16,fp8,0,5.327563095092773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,1,128,1,fp8,fp8,0,5.358531188964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,2,128,1,bfloat16,fp8,0,5.505174255371093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,1,128,1,bfloat16,bfloat16,0,6.231024169921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,2,128,1,bfloat16,bfloat16,0,6.181417465209961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,2,128,1,fp8,fp8,0,5.295816040039062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,4,128,1,bfloat16,fp8,0,5.371454238891602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,4,128,1,bfloat16,bfloat16,0,6.5027107238769535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,24,128,1,bfloat16,bfloat16,0,3.4694896697998048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,24,128,1,bfloat16,fp8,0,2.929475212097168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,4,128,1,fp8,fp8,0,5.633612823486328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,24,128,1,fp8,fp8,0,2.9334543228149412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,8,128,1,bfloat16,fp8,0,5.596129608154297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,8,128,1,fp8,fp8,0,5.454134368896485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,1,128,1,bfloat16,bfloat16,0,3.0898303985595703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,24,8,128,1,bfloat16,bfloat16,0,7.103727722167969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,1,128,1,bfloat16,fp8,0,2.67117919921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,1,128,1,fp8,fp8,0,2.6264671325683593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,2,128,1,bfloat16,fp8,0,2.6426368713378907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,2,128,1,bfloat16,bfloat16,0,3.0953039169311523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,2,128,1,fp8,fp8,0,2.6914159774780275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,4,128,1,bfloat16,bfloat16,0,3.118244743347168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,4,128,1,bfloat16,fp8,0,2.7490367889404297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,4,128,1,fp8,fp8,0,2.675396728515625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,8,128,1,bfloat16,bfloat16,0,2.9709600448608398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,24,128,1,bfloat16,bfloat16,0,1.6893951416015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,24,128,1,bfloat16,fp8,0,1.6341503143310547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,8,128,1,bfloat16,fp8,0,2.708305549621582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,24,128,1,fp8,fp8,0,1.47706880569458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,1,128,1,bfloat16,bfloat16,0,1.3931568145751954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,24,8,128,1,fp8,fp8,0,2.973316764831543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,1,128,1,bfloat16,fp8,0,1.4809103965759278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,1,128,1,fp8,fp8,0,1.5390928268432618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,2,128,1,bfloat16,bfloat16,0,1.4787440299987793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,2,128,1,bfloat16,fp8,0,1.3206512451171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,2,128,1,fp8,fp8,0,1.3614800453186036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,4,128,1,bfloat16,bfloat16,0,1.6004655838012696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,4,128,1,bfloat16,fp8,0,1.413907241821289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,4,128,1,fp8,fp8,0,1.406668758392334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,24,128,1,bfloat16,bfloat16,0,0.8573408126831055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,8,128,1,bfloat16,bfloat16,0,1.532915210723877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,24,128,1,fp8,fp8,0,0.7612063884735107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,8,128,1,fp8,fp8,0,1.3547167778015137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,24,128,1,bfloat16,fp8,0,1.0190159797668457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,24,8,128,1,bfloat16,fp8,0,1.5967535972595215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,1,128,1,bfloat16,bfloat16,0,0.7055840015411377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,1,128,1,bfloat16,fp8,0,0.672705602645874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,1,128,1,fp8,fp8,0,0.7923408031463623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,2,128,1,bfloat16,bfloat16,0,0.7766960144042969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,2,128,1,fp8,fp8,0,0.6704624176025391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,4,128,1,bfloat16,bfloat16,0,0.8683263778686523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,4,128,1,bfloat16,fp8,0,0.7673039913177491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,4,128,1,fp8,fp8,0,0.6956448078155517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,24,128,1,bfloat16,bfloat16,0,0.42751522064208985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,8,128,1,bfloat16,bfloat16,0,0.7574096202850342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,8,128,1,fp8,fp8,0,0.7561872005462646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,1,128,1,bfloat16,bfloat16,0,0.36056320667266845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,24,128,1,bfloat16,fp8,0,0.46517119407653806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,1,128,1,bfloat16,fp8,0,0.33684160709381106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,1,128,1,fp8,fp8,0,0.3418895959854126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,2,128,1,bfloat16,bfloat16,0,0.37411839962005616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,2,128,1,bfloat16,fp8,0,0.3475759983062744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,2,128,1,fp8,fp8,0,0.42383837699890137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,4,128,1,bfloat16,fp8,0,0.3484208106994629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,4,128,1,fp8,fp8,0,0.3622623920440674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,8,128,1,bfloat16,bfloat16,0,0.381715202331543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,8,128,1,bfloat16,fp8,0,0.4346640110015869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,8,128,1,fp8,fp8,0,0.3544975996017456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,24,128,1,bfloat16,bfloat16,0,0.2275167942047119
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,24,128,1,fp8,fp8,0,0.22690720558166505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,1,128,1,bfloat16,bfloat16,0,0.18959039449691772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,1,128,1,bfloat16,fp8,0,0.1807536005973816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,1,128,1,fp8,fp8,0,0.1796831965446472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,2,128,1,bfloat16,bfloat16,0,0.19094400405883788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,2,128,1,bfloat16,fp8,0,0.1803696036338806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,2,128,1,fp8,fp8,0,0.1754591941833496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,4,128,1,bfloat16,bfloat16,0,0.19475680589675903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,4,128,1,bfloat16,fp8,0,0.18165919780731202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,4,128,1,fp8,fp8,0,0.17721760272979736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,8,128,1,bfloat16,bfloat16,0,0.1999824047088623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,8,128,1,bfloat16,fp8,0,0.18514879941940307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,8,128,1,fp8,fp8,0,0.1808287978172302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,2,128,1,bfloat16,fp8,0,0.6700304031372071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,24,8,128,1,bfloat16,fp8,0,0.7637440204620362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,24,128,1,fp8,fp8,0,0.4066175937652588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,24,4,128,1,bfloat16,bfloat16,0,0.37180960178375244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,24,24,128,1,bfloat16,fp8,0,0.19933120012283326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,1,128,1,bfloat16,fp8,0,3.173387145996094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,1,128,1,fp8,fp8,0,3.1158592224121096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,1,128,1,bfloat16,bfloat16,0,3.6431648254394533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,2,128,1,bfloat16,bfloat16,0,3.4077632904052733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,2,128,1,bfloat16,fp8,0,3.13845272064209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,2,128,1,fp8,fp8,0,3.1343135833740234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,4,128,1,bfloat16,bfloat16,0,3.473654556274414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,4,128,1,bfloat16,fp8,0,3.216563034057617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,24,128,1,bfloat16,fp8,0,1.7994831085205079
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,4,128,1,fp8,fp8,0,3.1820255279541017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,24,128,1,bfloat16,bfloat16,0,2.0970287322998047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,8,128,1,bfloat16,bfloat16,0,3.647449493408203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,8,128,1,bfloat16,fp8,0,3.461975860595703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,24,8,128,1,fp8,fp8,0,3.505998229980469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,1,128,1,bfloat16,bfloat16,0,1.6995040893554687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,24,128,1,fp8,fp8,0,1.9516672134399413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,1,128,1,bfloat16,fp8,0,1.5634832382202148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,1,128,1,fp8,fp8,0,1.5585935592651368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,2,128,1,bfloat16,bfloat16,0,1.8521999359130858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,2,128,1,fp8,fp8,0,1.573697566986084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,2,128,1,bfloat16,fp8,0,1.8794368743896483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,4,128,1,bfloat16,bfloat16,0,1.7432847976684571
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,4,128,1,fp8,fp8,0,1.6299264907836915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,4,128,1,bfloat16,fp8,0,1.7526687622070312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,8,128,1,bfloat16,fp8,0,1.629867172241211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,24,128,1,bfloat16,bfloat16,0,1.0576111793518066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,24,128,1,bfloat16,fp8,0,0.9333968162536621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,8,128,1,bfloat16,bfloat16,0,1.8283103942871093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,1,128,1,bfloat16,bfloat16,0,0.8223615646362304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,1,128,1,bfloat16,fp8,0,0.8113535881042481
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,24,128,1,fp8,fp8,0,1.0068991661071778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,24,8,128,1,fp8,fp8,0,1.7616943359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,1,128,1,fp8,fp8,0,0.7876239776611328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,2,128,1,bfloat16,fp8,0,0.7923984050750732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,2,128,1,bfloat16,bfloat16,0,0.9139920234680176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,2,128,1,fp8,fp8,0,0.8616703987121582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,4,128,1,bfloat16,bfloat16,0,0.861137580871582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,4,128,1,bfloat16,fp8,0,0.8758879661560058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,8,128,1,bfloat16,fp8,0,0.8235263824462891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,24,128,1,bfloat16,fp8,0,0.5015007972717285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,24,128,1,bfloat16,bfloat16,0,0.5359072208404541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,24,128,1,fp8,fp8,0,0.47865920066833495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,1,128,1,bfloat16,bfloat16,0,0.4147776126861572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,1,128,1,bfloat16,fp8,0,0.4213551998138428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,1,128,1,fp8,fp8,0,0.40636639595031737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,2,128,1,bfloat16,bfloat16,0,0.43436479568481445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,2,128,1,bfloat16,fp8,0,0.4086656093597412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,2,128,1,fp8,fp8,0,0.42401762008666993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,4,128,1,bfloat16,bfloat16,0,0.43952479362487795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,4,128,1,bfloat16,fp8,0,0.41760802268981934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,4,128,1,fp8,fp8,0,0.4156976222991943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,8,128,1,bfloat16,bfloat16,0,0.4568624019622803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,8,128,1,fp8,fp8,0,0.4257967948913574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,24,128,1,bfloat16,bfloat16,0,0.2738976001739502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,24,128,1,bfloat16,fp8,0,0.24173600673675538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,24,128,1,fp8,fp8,0,0.23959040641784668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,1,128,1,bfloat16,bfloat16,0,0.2223871946334839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,1,128,1,bfloat16,fp8,0,0.2118016004562378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,2,128,1,bfloat16,bfloat16,0,0.2212399959564209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,2,128,1,bfloat16,fp8,0,0.2116528034210205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,2,128,1,fp8,fp8,0,0.21359679698944092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,4,128,1,bfloat16,bfloat16,0,0.22572638988494872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,4,128,1,bfloat16,fp8,0,0.21538879871368408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,4,128,1,fp8,fp8,0,0.815062427520752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,4,128,1,fp8,fp8,0,0.21488959789276124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,8,128,1,bfloat16,bfloat16,0,0.23281919956207275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,8,128,1,bfloat16,fp8,0,0.21571519374847412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,8,128,1,fp8,fp8,0,0.22030398845672608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,24,128,1,bfloat16,bfloat16,0,0.14723039865493776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,24,128,1,bfloat16,fp8,0,0.12497600317001342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,8,128,1,fp8,fp8,0,0.901636791229248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,1,128,1,bfloat16,bfloat16,0,0.11769280433654786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,1,128,1,bfloat16,fp8,0,0.10687999725341797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,1,128,1,fp8,fp8,0,0.10947840213775635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,2,128,1,bfloat16,fp8,0,0.10718239545822143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,2,128,1,fp8,fp8,0,0.10936959981918334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,4,128,1,bfloat16,bfloat16,0,0.12057440280914307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,4,128,1,bfloat16,fp8,0,0.11179519891738891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,4,128,1,fp8,fp8,0,0.10920640230178832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,8,128,1,bfloat16,bfloat16,0,0.125819194316864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,8,128,1,bfloat16,fp8,0,0.11409599781036377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,8,128,1,fp8,fp8,0,0.11630719900131226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,24,8,128,1,bfloat16,fp8,0,0.4338496208190918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,24,1,128,1,fp8,fp8,0,0.20981760025024415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,24,8,128,1,bfloat16,bfloat16,0,0.9062496185302734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,24,128,1,fp8,fp8,0,0.12660319805145265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,24,2,128,1,bfloat16,bfloat16,0,0.11952799558639526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,1,128,1,bfloat16,fp8,0,3.092238426208496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,1,128,1,fp8,fp8,0,3.1232160568237304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,1,128,1,bfloat16,bfloat16,0,3.3538543701171877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,2,128,1,bfloat16,bfloat16,0,3.319113540649414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,2,128,1,bfloat16,fp8,0,3.110887908935547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,2,128,1,fp8,fp8,0,3.109828758239746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,4,128,1,bfloat16,bfloat16,0,3.540897750854492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,4,128,1,fp8,fp8,0,3.1576160430908202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,24,128,1,bfloat16,bfloat16,0,2.1524032592773437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,8,128,1,bfloat16,fp8,0,3.262579345703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,8,128,1,bfloat16,bfloat16,0,3.799203109741211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,8,128,1,fp8,fp8,0,3.433135986328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,24,128,1,bfloat16,fp8,0,1.845795249938965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,1,128,1,bfloat16,bfloat16,0,1.6286575317382812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,24,128,1,fp8,fp8,0,1.9697343826293945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,1,128,1,bfloat16,fp8,0,1.5978927612304688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,1,128,1,fp8,fp8,0,1.536787223815918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,2,128,1,bfloat16,bfloat16,0,1.7554096221923827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,2,128,1,bfloat16,fp8,0,1.5469535827636718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,2,128,1,fp8,fp8,0,1.5400495529174805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,24,4,128,1,bfloat16,fp8,0,3.164900779724121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,4,128,1,bfloat16,fp8,0,1.5777600288391114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,4,128,1,fp8,fp8,0,1.6029312133789062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,8,128,1,bfloat16,bfloat16,0,1.7809295654296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,24,128,1,bfloat16,bfloat16,0,1.0747311592102051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,24,128,1,bfloat16,fp8,0,0.9301535606384277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,8,128,1,bfloat16,fp8,0,1.7819215774536132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,8,128,1,fp8,fp8,0,1.6227935791015624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,1,128,1,bfloat16,bfloat16,0,0.8069711685180664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,24,128,1,fp8,fp8,0,0.930720043182373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,1,128,1,bfloat16,fp8,0,0.8790207862854004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,1,128,1,fp8,fp8,0,0.7744383811950684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,2,128,1,bfloat16,fp8,0,0.7850080013275147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,2,128,1,bfloat16,bfloat16,0,0.8035231590270996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,2,128,1,fp8,fp8,0,0.7806399822235107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,4,128,1,bfloat16,fp8,0,0.7955920219421386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,4,128,1,bfloat16,bfloat16,0,0.8780912399291992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,4,128,1,fp8,fp8,0,0.8378303527832032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,8,128,1,bfloat16,bfloat16,0,0.8787424087524414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,24,128,1,bfloat16,bfloat16,0,0.5405471801757813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,24,128,1,fp8,fp8,0,0.4955872058868408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,8,128,1,bfloat16,fp8,0,0.8180031776428223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,24,8,128,1,fp8,fp8,0,0.8682671546936035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,1,128,1,bfloat16,bfloat16,0,0.41556482315063475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,1,128,1,bfloat16,fp8,0,0.43942718505859374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,1,128,1,fp8,fp8,0,0.3928832054138184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,2,128,1,bfloat16,bfloat16,0,0.4141439914703369
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,2,128,1,bfloat16,fp8,0,0.41921119689941405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,2,128,1,fp8,fp8,0,0.40800161361694337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,4,128,1,bfloat16,bfloat16,0,0.44182238578796384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,4,128,1,bfloat16,fp8,0,0.40366878509521487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,24,4,128,1,bfloat16,bfloat16,0,1.8748224258422852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,4,128,1,fp8,fp8,0,0.4019775867462158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,8,128,1,bfloat16,bfloat16,0,0.4580399990081787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,8,128,1,bfloat16,fp8,0,0.42763519287109375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,24,128,1,bfloat16,bfloat16,0,0.2777120113372803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,24,128,1,bfloat16,fp8,0,0.2446592092514038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,24,128,1,fp8,fp8,0,0.24278879165649414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,1,128,1,bfloat16,bfloat16,0,0.21677279472351074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,1,128,1,bfloat16,fp8,0,0.20583200454711914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,1,128,1,fp8,fp8,0,0.20460638999938965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,2,128,1,bfloat16,bfloat16,0,0.21281919479370118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,2,128,1,bfloat16,fp8,0,0.2029263973236084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,2,128,1,fp8,fp8,0,0.213102388381958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,4,128,1,bfloat16,bfloat16,0,0.21731998920440673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,4,128,1,bfloat16,fp8,0,0.21196320056915283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,4,128,1,fp8,fp8,0,0.20872800350189208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,8,128,1,bfloat16,bfloat16,0,0.22972800731658935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,8,128,1,bfloat16,fp8,0,0.22428319454193116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,24,8,128,1,fp8,fp8,0,0.21444640159606934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,24,128,1,bfloat16,fp8,0,0.12807199954986573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,24,128,1,fp8,fp8,0,0.1267680048942566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,1,128,1,bfloat16,bfloat16,0,0.11540319919586181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,1,128,1,bfloat16,fp8,0,0.10612479448318482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,1,128,1,fp8,fp8,0,0.10682560205459594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,2,128,1,bfloat16,bfloat16,0,0.11451200246810914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,2,128,1,bfloat16,fp8,0,0.11060800552368164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,2,128,1,fp8,fp8,0,0.10719039440155029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,4,128,1,bfloat16,bfloat16,0,0.1155616044998169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,4,128,1,bfloat16,fp8,0,0.11007519960403442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,4,128,1,fp8,fp8,0,0.11223039627075196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,8,128,1,bfloat16,bfloat16,0,0.11995359659194946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,8,128,1,bfloat16,fp8,0,0.11325759887695312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,8,128,1,fp8,fp8,0,0.11389759778976441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,24,128,1,bfloat16,bfloat16,0,0.08652639985084534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,24,128,1,bfloat16,fp8,0,0.07333279848098755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,24,128,1,fp8,fp8,0,0.0734224021434784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,1,128,1,bfloat16,bfloat16,0,0.06061599850654602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,1,128,1,bfloat16,fp8,0,0.05911840200424194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,1,128,1,fp8,fp8,0,0.05745599865913391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,2,128,1,bfloat16,bfloat16,0,0.059956800937652585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,2,128,1,bfloat16,fp8,0,0.05891680121421814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,2,128,1,fp8,fp8,0,0.05906559824943543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,4,128,1,bfloat16,bfloat16,0,0.0608784019947052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,4,128,1,bfloat16,fp8,0,0.0587007999420166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,4,128,1,fp8,fp8,0,0.0590719997882843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,8,128,1,bfloat16,bfloat16,0,0.06793439984321595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,8,128,1,fp8,fp8,0,0.06062560081481934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,24,128,1,bfloat16,fp8,0,0.4772463798522949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,24,8,128,1,fp8,fp8,0,0.4337024211883545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,1,128,1,bfloat16,fp8,0,1.9343791961669923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,1,128,1,fp8,fp8,0,1.9258415222167968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,2,128,1,bfloat16,bfloat16,0,1.9789152145385742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,24,24,128,1,bfloat16,bfloat16,0,0.1491055965423584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,24,8,128,1,bfloat16,fp8,0,0.05939040184020996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,2,128,1,fp8,fp8,0,1.9061552047729493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,4,128,1,bfloat16,bfloat16,0,2.057441520690918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,4,128,1,bfloat16,fp8,0,1.9956432342529298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,4,128,1,fp8,fp8,0,1.9717199325561523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,1,128,1,bfloat16,bfloat16,0,2.0129119873046877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,24,128,1,bfloat16,fp8,0,1.2019712448120117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,24,128,1,bfloat16,bfloat16,0,1.4279295921325683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,8,128,1,bfloat16,fp8,0,2.0594976425170897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,24,128,1,fp8,fp8,0,1.194644832611084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,8,128,1,bfloat16,bfloat16,0,2.211675262451172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,8,128,1,fp8,fp8,0,2.0363759994506836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,1,128,1,bfloat16,bfloat16,0,0.9928943634033203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,1,128,1,bfloat16,fp8,0,0.9631088256835938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,2,128,1,bfloat16,bfloat16,0,0.9936415672302246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,2,128,1,fp8,fp8,0,1.0574352264404296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,24,2,128,1,bfloat16,fp8,0,1.908515167236328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,4,128,1,bfloat16,bfloat16,0,1.0186944007873535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,4,128,1,bfloat16,fp8,0,1.0386719703674316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,24,128,1,bfloat16,bfloat16,0,0.6967872142791748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,8,128,1,bfloat16,fp8,0,1.0731776237487793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,8,128,1,fp8,fp8,0,1.025822353363037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,24,128,1,bfloat16,fp8,0,0.6333360195159912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,24,128,1,fp8,fp8,0,0.6095871925354004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,1,128,1,bfloat16,bfloat16,0,0.4962607860565186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,1,128,1,bfloat16,fp8,0,0.4830751895904541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,1,128,1,fp8,fp8,0,0.4965023994445801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,2,128,1,bfloat16,fp8,0,0.4916399955749512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,4,128,1,bfloat16,bfloat16,0,0.525491189956665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,4,128,1,bfloat16,fp8,0,0.49979357719421386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,4,128,1,fp8,fp8,0,0.4980368137359619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,2,128,1,bfloat16,fp8,0,0.9600496292114258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,8,128,1,bfloat16,bfloat16,0,0.5495312213897705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,8,128,1,bfloat16,fp8,0,0.5184688091278076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,8,128,1,fp8,fp8,0,0.5160704135894776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,24,128,1,bfloat16,bfloat16,0,0.35382080078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,24,128,1,bfloat16,fp8,0,0.311409592628479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,4,128,1,fp8,fp8,0,0.9840736389160156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,1,128,1,bfloat16,bfloat16,0,0.2521424055099487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,24,128,1,fp8,fp8,0,0.31049280166625975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,8,128,1,bfloat16,bfloat16,0,1.1364496231079102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,1,128,1,bfloat16,fp8,0,0.24708640575408936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,1,128,1,fp8,fp8,0,0.24697599411010743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,2,128,1,bfloat16,bfloat16,0,0.2543839931488037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,2,128,1,fp8,fp8,0,0.24925599098205567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,2,128,1,bfloat16,fp8,0,0.2547568082809448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,4,128,1,bfloat16,bfloat16,0,0.2641247987747192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,4,128,1,bfloat16,fp8,0,0.25348639488220215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,4,128,1,fp8,fp8,0,0.25557920932769773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,8,128,1,bfloat16,bfloat16,0,0.2884655952453613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,8,128,1,bfloat16,fp8,0,0.2700799942016602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,24,8,128,1,fp8,fp8,0,0.26425919532775877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,24,128,1,bfloat16,bfloat16,0,0.18624160289764405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,24,128,1,bfloat16,fp8,0,0.16815359592437745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,24,128,1,fp8,fp8,0,0.16780320405960084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,1,128,1,bfloat16,bfloat16,0,0.13193919658660888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,1,128,1,bfloat16,fp8,0,0.1296656012535095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,1,128,1,fp8,fp8,0,0.13252320289611816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,2,128,1,bfloat16,bfloat16,0,0.5018928050994873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,2,128,1,bfloat16,bfloat16,0,0.1368656039237976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,2,128,1,bfloat16,fp8,0,0.13005599975585938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,2,128,1,fp8,fp8,0,0.13093760013580322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,4,128,1,bfloat16,bfloat16,0,0.14223840236663818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,4,128,1,bfloat16,fp8,0,0.136135995388031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,4,128,1,fp8,fp8,0,0.13604960441589356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,8,128,1,bfloat16,bfloat16,0,0.14553439617156982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,8,128,1,bfloat16,fp8,0,0.13929599523544312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,24,8,128,1,fp8,fp8,0,0.14218239784240722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,24,128,1,bfloat16,bfloat16,0,0.10036319494247437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,24,128,1,fp8,fp8,0,0.09148319959640502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,1,128,1,bfloat16,bfloat16,0,0.07208160161972046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,1,128,1,bfloat16,fp8,0,0.06946560144424438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,1,128,1,fp8,fp8,0,0.06914079785346985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,2,128,1,bfloat16,bfloat16,0,0.07347840070724487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,2,128,1,fp8,fp8,0,0.0694432020187378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,4,128,1,bfloat16,bfloat16,0,0.07804480195045471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,4,128,1,bfloat16,fp8,0,0.06914560198783874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,4,128,1,fp8,fp8,0,0.07184320092201232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,8,128,1,bfloat16,bfloat16,0,0.08276320099830628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,8,128,1,bfloat16,fp8,0,0.07398560047149658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,24,1,128,1,fp8,fp8,0,1.019206428527832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,24,128,1,bfloat16,bfloat16,0,0.06060000061988831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,24,128,1,bfloat16,fp8,0,0.049807998538017276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,24,128,1,fp8,fp8,0,0.052300798892974856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,1,128,1,bfloat16,fp8,0,0.03926720023155213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,1,128,1,fp8,fp8,0,0.040854400396347045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,2,128,1,bfloat16,bfloat16,0,0.04037440121173859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,2,128,1,bfloat16,fp8,0,0.04094400107860565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,2,128,1,fp8,fp8,0,0.0397599995136261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,4,128,1,bfloat16,bfloat16,0,0.042438399791717527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,4,128,1,bfloat16,fp8,0,0.039673599600791934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,4,128,1,fp8,fp8,0,0.04122560024261475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,8,128,1,bfloat16,bfloat16,0,0.043296000361442565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,8,128,1,bfloat16,fp8,0,0.04161919951438904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,8,128,1,fp8,fp8,0,0.040380799770355226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,24,2,128,1,fp8,fp8,0,0.4867680072784424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,1,128,1,bfloat16,bfloat16,0,2.0451616287231444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,1,128,1,bfloat16,fp8,0,2.0134159088134767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,24,128,1,bfloat16,fp8,0,0.09363200068473816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,2,128,1,bfloat16,fp8,0,0.06897280216217042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,24,8,128,1,fp8,fp8,0,0.07686079740524292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,24,1,128,1,bfloat16,bfloat16,0,0.041921600699424744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,1,128,1,fp8,fp8,0,1.9954399108886718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,2,128,1,bfloat16,bfloat16,0,2.078446388244629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,2,128,1,bfloat16,fp8,0,2.030603218078613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,2,128,1,fp8,fp8,0,2.0240928649902346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,4,128,1,bfloat16,bfloat16,0,2.1959392547607424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,4,128,1,bfloat16,fp8,0,2.1009775161743165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,4,128,1,fp8,fp8,0,2.113172721862793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,8,128,1,bfloat16,bfloat16,0,2.381735992431641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,24,128,1,bfloat16,fp8,0,1.327355194091797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,24,128,1,bfloat16,bfloat16,0,1.56463680267334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,8,128,1,bfloat16,fp8,0,2.192011260986328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,1,128,1,bfloat16,fp8,0,1.021280002593994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,24,128,1,fp8,fp8,0,1.3496864318847657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,1,128,1,fp8,fp8,0,1.015214443206787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,24,8,128,1,fp8,fp8,0,2.220819282531738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,2,128,1,bfloat16,bfloat16,0,1.0339839935302735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,2,128,1,bfloat16,fp8,0,1.011171245574951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,4,128,1,bfloat16,bfloat16,0,1.0805536270141602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,4,128,1,bfloat16,fp8,0,1.042198371887207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,4,128,1,fp8,fp8,0,1.0421232223510741
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,8,128,1,bfloat16,bfloat16,0,1.1731103897094726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,24,128,1,bfloat16,bfloat16,0,0.7943503856658936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,24,128,1,bfloat16,fp8,0,0.6723120212554932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,8,128,1,bfloat16,fp8,0,1.134183979034424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,8,128,1,fp8,fp8,0,1.110035228729248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,24,128,1,fp8,fp8,0,0.6719696044921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,1,128,1,bfloat16,bfloat16,0,0.5239920139312744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,1,128,1,fp8,fp8,0,0.5084271907806397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,2,128,1,bfloat16,bfloat16,0,0.5353360176086426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,2,128,1,bfloat16,fp8,0,0.5211775779724122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,2,128,1,fp8,fp8,0,0.5108751773834228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,4,128,1,bfloat16,bfloat16,0,0.5498432159423828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,4,128,1,bfloat16,fp8,0,0.5283631801605224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,4,128,1,fp8,fp8,0,0.5370992183685303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,1,128,1,bfloat16,bfloat16,0,1.0219967842102051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,8,128,1,bfloat16,fp8,0,0.5532032012939453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,8,128,1,bfloat16,bfloat16,0,0.6042751789093017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,8,128,1,fp8,fp8,0,0.5598720073699951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,24,128,1,bfloat16,bfloat16,0,0.4114448070526123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,24,128,1,bfloat16,fp8,0,0.3497503995895386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,1,128,1,bfloat16,bfloat16,0,0.26759040355682373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,24,128,1,fp8,fp8,0,0.3417104005813599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,1,128,1,bfloat16,fp8,0,0.2690576076507568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,1,128,1,fp8,fp8,0,0.2655855894088745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,2,128,1,bfloat16,bfloat16,0,0.27405600547790526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,2,128,1,bfloat16,fp8,0,0.2670687913894653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,2,128,1,fp8,fp8,0,0.2596544027328491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,24,2,128,1,fp8,fp8,0,1.0004143714904785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,4,128,1,bfloat16,bfloat16,0,0.28933761119842527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,4,128,1,bfloat16,fp8,0,0.2757519960403442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,4,128,1,fp8,fp8,0,0.275598406791687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,8,128,1,bfloat16,bfloat16,0,0.3086863994598389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,8,128,1,bfloat16,fp8,0,0.28331680297851564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,24,128,1,bfloat16,fp8,0,0.18411999940872192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,24,128,1,bfloat16,bfloat16,0,0.21533920764923095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,24,128,1,fp8,fp8,0,0.18389439582824707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,1,128,1,bfloat16,bfloat16,0,0.14167840480804444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,24,8,128,1,fp8,fp8,0,0.28569118976593016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,1,128,1,bfloat16,fp8,0,0.1352352023124695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,1,128,1,fp8,fp8,0,0.13946720361709594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,2,128,1,bfloat16,bfloat16,0,0.14526400566101075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,2,128,1,bfloat16,fp8,0,0.1404960036277771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,2,128,1,fp8,fp8,0,0.1407039999961853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,4,128,1,bfloat16,bfloat16,0,0.1471168041229248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,4,128,1,bfloat16,fp8,0,0.13988959789276123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,4,128,1,fp8,fp8,0,0.14451520442962645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,8,128,1,bfloat16,fp8,0,0.1526368021965027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,8,128,1,bfloat16,bfloat16,0,0.16092000007629395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,24,8,128,1,fp8,fp8,0,0.15292960405349731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,24,128,1,fp8,fp8,0,0.10056159496307374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,1,128,1,bfloat16,bfloat16,0,0.07883840203285217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,1,128,1,bfloat16,fp8,0,0.07506719827651978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,1,128,1,fp8,fp8,0,0.07551680207252502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,2,128,1,bfloat16,bfloat16,0,0.08028640151023865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,2,128,1,bfloat16,fp8,0,0.07671840190887451
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,2,128,1,fp8,fp8,0,0.07600319981575013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,4,128,1,bfloat16,bfloat16,0,0.08220319747924805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,4,128,1,bfloat16,fp8,0,0.07928000092506408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,4,128,1,fp8,fp8,0,0.07881280183792114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,8,128,1,bfloat16,bfloat16,0,0.08772479891777038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,8,128,1,bfloat16,fp8,0,0.08281120061874389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,8,128,1,fp8,fp8,0,0.08326719999313355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,24,128,1,bfloat16,bfloat16,0,0.06687359809875489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,24,1,128,1,bfloat16,fp8,0,0.5098159790039063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,24,128,1,bfloat16,fp8,0,0.05820639729499817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,24,128,1,fp8,fp8,0,0.05931360125541687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,1,128,1,bfloat16,fp8,0,0.04106239974498749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,2,128,1,bfloat16,bfloat16,0,0.04098080098628998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,2,128,1,bfloat16,fp8,0,0.041310399770736694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,2,128,1,fp8,fp8,0,0.0414112001657486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,4,128,1,bfloat16,bfloat16,0,0.04235840141773224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,4,128,1,bfloat16,fp8,0,0.04264479875564575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,4,128,1,fp8,fp8,0,0.04280959963798523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,8,128,1,bfloat16,bfloat16,0,0.047056001424789426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,8,128,1,bfloat16,fp8,0,0.044007998704910276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,8,128,1,fp8,fp8,0,0.043849599361419675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,24,128,1,bfloat16,bfloat16,0,0.03635999858379364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,24,128,1,bfloat16,fp8,0,0.03206560015678406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,24,128,1,fp8,fp8,0,0.032148799300193785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,1,128,1,bfloat16,bfloat16,0,0.02417919933795929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,1,128,1,bfloat16,fp8,0,0.025198400020599365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,1,128,1,fp8,fp8,0,0.024879999458789825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,2,128,1,bfloat16,bfloat16,0,0.02452960014343262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,2,128,1,bfloat16,fp8,0,0.025430399179458617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,2,128,1,fp8,fp8,0,0.024987199902534486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,4,128,1,bfloat16,bfloat16,0,0.024694399535655977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,4,128,1,bfloat16,fp8,0,0.026678401231765746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,4,128,1,fp8,fp8,0,0.02662239968776703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,8,128,1,bfloat16,bfloat16,0,0.025595200061798096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,8,128,1,bfloat16,fp8,0,0.026844799518585205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,1,128,1,bfloat16,bfloat16,0,1.4274592399597168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,1,128,1,bfloat16,fp8,0,1.4690560340881347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,24,128,1,bfloat16,bfloat16,0,0.11340639591217042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,24,24,128,1,bfloat16,fp8,0,0.09747040271759033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,1,128,1,bfloat16,bfloat16,0,0.04059840142726898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,24,1,128,1,fp8,fp8,0,0.04089120030403137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,1,128,1,fp8,fp8,0,1.4522064208984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,2,128,1,bfloat16,bfloat16,0,1.4758496284484863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,2,128,1,bfloat16,fp8,0,1.454643154144287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,4,128,1,bfloat16,bfloat16,0,1.5431743621826173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,24,8,128,1,fp8,fp8,0,0.026929599046707154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,4,128,1,bfloat16,fp8,0,1.5235055923461913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,4,128,1,fp8,fp8,0,1.5433695793151856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,8,128,1,bfloat16,bfloat16,0,1.732499122619629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,24,128,1,bfloat16,fp8,0,1.0837424278259278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,8,128,1,bfloat16,fp8,0,1.6653839111328126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,8,128,1,fp8,fp8,0,1.642959976196289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,1,128,1,bfloat16,bfloat16,0,0.7236656188964844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,24,128,1,fp8,fp8,0,1.0734031677246094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,1,128,1,bfloat16,fp8,0,0.7292384147644043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,1,128,1,fp8,fp8,0,0.7307295799255371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,2,128,1,bfloat16,bfloat16,0,0.7454463958740234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,2,128,1,bfloat16,fp8,0,0.7297183990478515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,2,128,1,fp8,fp8,0,0.7461904048919678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,4,128,1,bfloat16,bfloat16,0,0.7841968059539794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,4,128,1,bfloat16,fp8,0,0.7736048221588134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,4,128,1,fp8,fp8,0,0.768782377243042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,24,2,128,1,fp8,fp8,0,1.4689807891845703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,8,128,1,bfloat16,bfloat16,0,0.8847439765930176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,8,128,1,bfloat16,fp8,0,0.8234288215637207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,8,128,1,fp8,fp8,0,0.8300576210021973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,24,128,1,bfloat16,bfloat16,0,0.6501935958862305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,24,128,1,bfloat16,fp8,0,0.5446767807006836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,1,128,1,bfloat16,bfloat16,0,0.369270396232605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,24,128,1,fp8,fp8,0,0.5442560195922852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,1,128,1,bfloat16,fp8,0,0.36858720779418946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,1,128,1,fp8,fp8,0,0.37093119621276854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,2,128,1,bfloat16,bfloat16,0,0.3789263963699341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,2,128,1,fp8,fp8,0,0.38027520179748536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,4,128,1,bfloat16,bfloat16,0,0.3973567962646484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,4,128,1,bfloat16,fp8,0,0.3880592107772827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,4,128,1,fp8,fp8,0,0.3902656078338623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,8,128,1,bfloat16,bfloat16,0,0.4471744060516357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,24,24,128,1,bfloat16,bfloat16,0,1.2743552207946778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,8,128,1,bfloat16,fp8,0,0.42657761573791503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,24,128,1,bfloat16,fp8,0,0.27930240631103515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,24,128,1,bfloat16,bfloat16,0,0.3298192024230957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,8,128,1,fp8,fp8,0,0.42111358642578123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,24,128,1,fp8,fp8,0,0.2811919927597046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,1,128,1,bfloat16,bfloat16,0,0.19154880046844483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,2,128,1,bfloat16,bfloat16,0,0.1964687943458557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,2,128,1,bfloat16,fp8,0,0.19360480308532715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,2,128,1,fp8,fp8,0,0.1956480026245117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,4,128,1,bfloat16,bfloat16,0,0.20755839347839355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,4,128,1,bfloat16,fp8,0,0.20384318828582765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,4,128,1,fp8,fp8,0,0.20195999145507812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,8,128,1,bfloat16,bfloat16,0,0.23122239112854004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,8,128,1,bfloat16,fp8,0,0.22014880180358887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,24,128,1,bfloat16,bfloat16,0,0.17319999933242797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,8,128,1,fp8,fp8,0,0.21879680156707765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,24,128,1,bfloat16,fp8,0,0.14789119958877564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,24,128,1,fp8,fp8,0,0.14870719909667968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,1,128,1,bfloat16,fp8,0,0.10285760164260864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,1,128,1,fp8,fp8,0,0.10263680219650269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,2,128,1,bfloat16,bfloat16,0,0.10528160333633423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,2,128,1,bfloat16,fp8,0,0.10397440195083618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,2,128,1,fp8,fp8,0,0.10375200510025025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,4,128,1,bfloat16,bfloat16,0,0.11089279651641845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,4,128,1,bfloat16,fp8,0,0.1075711965560913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,4,128,1,fp8,fp8,0,0.10788320302963257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,8,128,1,bfloat16,bfloat16,0,0.12137279510498047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,8,128,1,bfloat16,fp8,0,0.1159440040588379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,8,128,1,fp8,fp8,0,0.11579840183258057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,24,128,1,bfloat16,bfloat16,0,0.09433760046958924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,24,128,1,bfloat16,fp8,0,0.08084800243377685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,1,128,1,bfloat16,bfloat16,0,0.057764798402786255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,1,128,1,bfloat16,fp8,0,0.05721279978752136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,1,128,1,fp8,fp8,0,0.056308799982070924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,2,128,1,bfloat16,bfloat16,0,0.05876799821853638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,2,128,1,bfloat16,fp8,0,0.05782880187034607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,2,128,1,fp8,fp8,0,0.05789600014686584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,4,128,1,bfloat16,bfloat16,0,0.06138719916343689
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,4,128,1,bfloat16,fp8,0,0.06026880145072937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,4,128,1,fp8,fp8,0,0.0603007972240448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,8,128,1,bfloat16,bfloat16,0,0.06700959801673889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,8,128,1,bfloat16,fp8,0,0.06483680009841919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,24,2,128,1,bfloat16,fp8,0,0.3747087955474854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,24,128,1,bfloat16,bfloat16,0,0.05380960106849671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,24,128,1,bfloat16,fp8,0,0.04639999866485596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,24,128,1,fp8,fp8,0,0.04697760045528412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,1,128,1,bfloat16,bfloat16,0,0.03171519935131073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,1,128,1,bfloat16,fp8,0,0.03139519989490509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,1,128,1,fp8,fp8,0,0.031065601110458373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,2,128,1,bfloat16,bfloat16,0,0.03264319896697998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,2,128,1,bfloat16,fp8,0,0.031012800335884095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,2,128,1,fp8,fp8,0,0.03140639960765838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,4,128,1,bfloat16,bfloat16,0,0.03295679986476898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,4,128,1,bfloat16,fp8,0,0.03272959887981415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,4,128,1,fp8,fp8,0,0.03200480043888092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,8,128,1,bfloat16,bfloat16,0,0.03945600092411041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,8,128,1,bfloat16,fp8,0,0.034560000896453856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,1,128,1,bfloat16,fp8,0,0.19488320350646973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,24,8,128,1,fp8,fp8,0,0.03445279896259308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,24,1,128,1,fp8,fp8,0,0.19156479835510254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,24,128,1,bfloat16,bfloat16,0,0.03135519921779632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,24,128,1,bfloat16,fp8,0,0.028145599365234374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,24,128,1,fp8,fp8,0,0.028244799375534056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,1,128,1,bfloat16,bfloat16,0,0.0204815998673439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,1,128,1,bfloat16,fp8,0,0.0205935999751091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,1,128,1,fp8,fp8,0,0.020550400018692017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,2,128,1,bfloat16,bfloat16,0,0.020979200303554536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,2,128,1,bfloat16,fp8,0,0.020107200741767882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,4,128,1,bfloat16,bfloat16,0,0.021480000019073485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,4,128,1,bfloat16,fp8,0,0.022540800273418427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,4,128,1,fp8,fp8,0,0.02196960002183914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,8,128,1,bfloat16,bfloat16,0,0.021934400498867034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,8,128,1,bfloat16,fp8,0,0.022945599257946016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,8,128,1,fp8,fp8,0,0.022286400198936462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,24,128,1,bfloat16,bfloat16,0,0.019556799530982973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,24,128,1,bfloat16,fp8,0,0.020095999538898467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,24,128,1,fp8,fp8,0,0.019625599682331085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,1,128,1,bfloat16,bfloat16,0,0.017523199319839478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,1,128,1,bfloat16,fp8,0,0.018668800592422485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,1,128,1,fp8,fp8,0,0.018411199748516082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,2,128,1,bfloat16,bfloat16,0,0.0175135999917984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,2,128,1,bfloat16,fp8,0,0.018782399594783783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,2,128,1,fp8,fp8,0,0.018848000466823576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,4,128,1,bfloat16,bfloat16,0,0.017895999550819396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,24,1,128,1,bfloat16,bfloat16,0,0.10124640464782715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,4,128,1,bfloat16,fp8,0,0.019196799397468566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,4,128,1,fp8,fp8,0,0.018987199664115904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,8,128,1,bfloat16,bfloat16,0,0.017995199561119078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,8,128,1,fp8,fp8,0,0.019289599359035493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,24,8,128,1,bfloat16,fp8,0,0.0193792000412941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,1,128,1,bfloat16,bfloat16,0,0.5738304138183594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,1,128,1,bfloat16,fp8,0,0.5978000164031982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,1,128,1,fp8,fp8,0,0.6097216129302978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,2,128,1,bfloat16,bfloat16,0,0.6013216018676758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,2,128,1,bfloat16,fp8,0,0.604472017288208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,2,128,1,fp8,fp8,0,0.6039216041564941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,4,128,1,bfloat16,bfloat16,0,0.6503727912902832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,4,128,1,bfloat16,fp8,0,0.6386528015136719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,8,128,1,fp8,fp8,0,0.0642799973487854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,4,128,1,fp8,fp8,0,0.6385087966918945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,8,128,1,bfloat16,bfloat16,0,0.7370096206665039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,8,128,1,bfloat16,fp8,0,0.708355188369751
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,24,8,128,1,fp8,fp8,0,0.7008031845092774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,24,2,128,1,fp8,fp8,0,0.020584000647068022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,1,128,1,bfloat16,bfloat16,0,0.2968287944793701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,1,128,1,bfloat16,fp8,0,0.3062671899795532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,24,128,1,fp8,fp8,0,0.45920958518981936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,1,128,1,fp8,fp8,0,0.3060192108154297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,2,128,1,bfloat16,bfloat16,0,0.3089263916015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,2,128,1,bfloat16,fp8,0,0.3096784114837646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,2,128,1,fp8,fp8,0,0.30951199531555174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,4,128,1,bfloat16,bfloat16,0,0.3280992031097412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,4,128,1,bfloat16,fp8,0,0.32636160850524903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,4,128,1,fp8,fp8,0,0.326911997795105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,8,128,1,bfloat16,bfloat16,0,0.3760783910751343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,8,128,1,bfloat16,fp8,0,0.3570703983306885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,8,128,1,fp8,fp8,0,0.3564464092254639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,24,128,1,bfloat16,bfloat16,0,0.29995360374450686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,24,128,1,bfloat16,fp8,0,0.2388319969177246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,24,24,128,1,fp8,fp8,0,0.08070560097694397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,1,128,1,bfloat16,bfloat16,0,0.1534127950668335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,24,128,1,fp8,fp8,0,0.23713760375976561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,1,128,1,bfloat16,fp8,0,0.15934560298919678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,1,128,1,fp8,fp8,0,0.1610751986503601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,2,128,1,bfloat16,bfloat16,0,0.15916800498962402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,2,128,1,bfloat16,fp8,0,0.16147520542144775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,2,128,1,fp8,fp8,0,0.16208319664001464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,4,128,1,bfloat16,bfloat16,0,0.17074400186538696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,4,128,1,bfloat16,fp8,0,0.17185759544372559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,4,128,1,fp8,fp8,0,0.16985759735107422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,8,128,1,bfloat16,bfloat16,0,0.19403040409088135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,8,128,1,bfloat16,fp8,0,0.18583359718322753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,24,128,1,bfloat16,bfloat16,0,0.15867359638214112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,24,128,1,fp8,fp8,0,0.12609920501708985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,1,128,1,bfloat16,bfloat16,0,0.08374559879302979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,1,128,1,bfloat16,fp8,0,0.08750560283660888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,1,128,1,fp8,fp8,0,0.08590720295906067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,2,128,1,bfloat16,bfloat16,0,0.08626559972763062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,2,128,1,bfloat16,fp8,0,0.08831359744071961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,2,128,1,fp8,fp8,0,0.08700479865074158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,4,128,1,bfloat16,bfloat16,0,0.09232320189476013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,4,128,1,bfloat16,fp8,0,0.09222559928894043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,4,128,1,fp8,fp8,0,0.09081760048866272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,8,128,1,bfloat16,bfloat16,0,0.10456960201263428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,24,128,1,bfloat16,bfloat16,0,0.5850671768188477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,8,128,1,bfloat16,fp8,0,0.10029759407043456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,8,128,1,fp8,fp8,0,0.0989520013332367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,24,128,1,bfloat16,bfloat16,0,0.0875760018825531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,24,128,1,bfloat16,fp8,0,0.06987040042877198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,24,128,1,fp8,fp8,0,0.06983839869499206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,1,128,1,bfloat16,fp8,0,0.049300798773765565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,2,128,1,bfloat16,bfloat16,0,0.05116159915924072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,1,128,1,fp8,fp8,0,0.04917280077934265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,2,128,1,bfloat16,fp8,0,0.050151997804641725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,2,128,1,fp8,fp8,0,0.05054559707641602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,24,24,128,1,bfloat16,fp8,0,0.4594831943511963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,4,128,1,bfloat16,bfloat16,0,0.05375999808311462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,4,128,1,bfloat16,fp8,0,0.051630401611328126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,4,128,1,fp8,fp8,0,0.05204799771308899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,8,128,1,bfloat16,bfloat16,0,0.059552001953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,8,128,1,fp8,fp8,0,0.056859201192855834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,8,128,1,bfloat16,fp8,0,0.05562559962272644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,24,128,1,bfloat16,bfloat16,0,0.04888319969177246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,24,128,1,bfloat16,fp8,0,0.03993119895458221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,24,128,1,fp8,fp8,0,0.04077920019626617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,1,128,1,bfloat16,bfloat16,0,0.027537599205970764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,1,128,1,bfloat16,fp8,0,0.027395200729370118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,1,128,1,fp8,fp8,0,0.0276528000831604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,2,128,1,bfloat16,bfloat16,0,0.02768639922142029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,2,128,1,bfloat16,fp8,0,0.028217598795890808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,2,128,1,fp8,fp8,0,0.028148800134658813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,4,128,1,fp8,fp8,0,0.028439998626708984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,4,128,1,bfloat16,fp8,0,0.028246399760246278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,8,128,1,bfloat16,fp8,0,0.03121120035648346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,8,128,1,bfloat16,bfloat16,0,0.034160000085830686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,8,128,1,fp8,fp8,0,0.0310591995716095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,24,128,1,bfloat16,fp8,0,0.022852799296379088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,1,128,1,bfloat16,bfloat16,0,0.019233599305152893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,1,128,1,fp8,fp8,0,0.020796799659729005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,2,128,1,bfloat16,bfloat16,0,0.01918880045413971
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,2,128,1,bfloat16,fp8,0,0.020326399803161622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,24,8,128,1,fp8,fp8,0,0.18528000116348267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,2,128,1,fp8,fp8,0,0.0202224001288414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,4,128,1,bfloat16,bfloat16,0,0.019393600523471832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,4,128,1,bfloat16,fp8,0,0.02060000002384186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,4,128,1,fp8,fp8,0,0.020638400316238405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,8,128,1,bfloat16,bfloat16,0,0.02022880017757416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,8,128,1,bfloat16,fp8,0,0.020900799334049223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,8,128,1,fp8,fp8,0,0.020955200493335723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,24,128,1,bfloat16,bfloat16,0,0.016771200299263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,24,128,1,bfloat16,fp8,0,0.016494399309158324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,24,1,128,1,bfloat16,bfloat16,0,0.04902079999446869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,24,128,1,fp8,fp8,0,0.01637600064277649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,1,128,1,bfloat16,bfloat16,0,0.014660799503326416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,1,128,1,bfloat16,fp8,0,0.015294399857521058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,1,128,1,fp8,fp8,0,0.01523360013961792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,2,128,1,bfloat16,bfloat16,0,0.014758400619029999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,2,128,1,bfloat16,fp8,0,0.015060800313949584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,2,128,1,fp8,fp8,0,0.015249599516391755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,4,128,1,bfloat16,bfloat16,0,0.014934399724006652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,4,128,1,bfloat16,fp8,0,0.01536639928817749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,4,128,1,fp8,fp8,0,0.015398399531841278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,8,128,1,bfloat16,bfloat16,0,0.015403200685977936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,8,128,1,bfloat16,fp8,0,0.015675200521945952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,24,4,128,1,bfloat16,bfloat16,0,0.02887679934501648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,24,8,128,1,fp8,fp8,0,0.015572799742221833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,24,128,1,bfloat16,bfloat16,0,0.015476800501346588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,24,128,1,bfloat16,fp8,0,0.01587039977312088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,24,128,1,fp8,fp8,0,0.01581760048866272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,1,128,1,bfloat16,bfloat16,0,0.014588800072669984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,1,128,1,bfloat16,fp8,0,0.01499360054731369
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,24,128,1,bfloat16,bfloat16,0,0.026387199759483337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,24,128,1,fp8,fp8,0,0.02282720059156418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,2,128,1,bfloat16,bfloat16,0,0.01440960019826889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,2,128,1,bfloat16,fp8,0,0.014934399724006652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,24,1,128,1,bfloat16,fp8,0,0.02061759978532791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,2,128,1,fp8,fp8,0,0.01505119949579239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,4,128,1,bfloat16,fp8,0,0.015371200442314149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,4,128,1,fp8,fp8,0,0.015358400344848634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,8,128,1,bfloat16,bfloat16,0,0.014827199280261993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,8,128,1,bfloat16,fp8,0,0.015486399829387664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,1,128,1,bfloat16,bfloat16,0,0.2839728116989136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,1,128,1,bfloat16,fp8,0,0.3029792070388794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,1,128,1,fp8,fp8,0,0.30448160171508787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,2,128,1,bfloat16,bfloat16,0,0.29247360229492186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,2,128,1,bfloat16,fp8,0,0.31653919219970705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,2,128,1,fp8,fp8,0,0.3161423921585083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,4,128,1,bfloat16,bfloat16,0,0.31345438957214355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,24,24,128,1,bfloat16,fp8,0,0.12634559869766235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,4,128,1,fp8,fp8,0,0.3341088056564331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,8,128,1,bfloat16,bfloat16,0,0.36006081104278564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,8,128,1,bfloat16,fp8,0,0.3662287950515747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,8,128,1,fp8,fp8,0,0.36607680320739744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,24,128,1,bfloat16,bfloat16,0,0.2996016025543213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,24,128,1,bfloat16,fp8,0,0.2472912073135376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,24,128,1,fp8,fp8,0,0.24907360076904297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,1,128,1,bfloat16,bfloat16,0,0.14704960584640503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,1,128,1,bfloat16,fp8,0,0.15515199899673462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,1,128,1,fp8,fp8,0,0.15705759525299073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,2,128,1,bfloat16,bfloat16,0,0.15215840339660644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,2,128,1,bfloat16,fp8,0,0.16357280015945436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,2,128,1,fp8,fp8,0,0.16349439620971679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,4,128,1,bfloat16,bfloat16,0,0.1629487991333008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,4,128,1,bfloat16,fp8,0,0.17317600250244142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,4,128,1,fp8,fp8,0,0.17397600412368774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,8,128,1,bfloat16,bfloat16,0,0.18578720092773438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,1,128,1,fp8,fp8,0,0.014876799285411834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,8,128,1,bfloat16,fp8,0,0.18984479904174806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,24,8,128,1,fp8,fp8,0,0.1906208038330078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,4,128,1,bfloat16,bfloat16,0,0.014921599626541137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,24,128,1,bfloat16,bfloat16,0,0.15930880308151246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,24,128,1,bfloat16,fp8,0,0.12896000146865844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,24,128,1,fp8,fp8,0,0.1288367986679077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,1,128,1,bfloat16,bfloat16,0,0.08009920120239258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,24,8,128,1,fp8,fp8,0,0.015545600652694702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,1,128,1,bfloat16,fp8,0,0.08524640202522278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,2,128,1,bfloat16,bfloat16,0,0.08399360179901123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,2,128,1,fp8,fp8,0,0.08719519972801208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,4,128,1,bfloat16,bfloat16,0,0.08942239880561828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,4,128,1,bfloat16,fp8,0,0.09067199826240539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,4,128,1,fp8,fp8,0,0.0900160014629364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,8,128,1,bfloat16,bfloat16,0,0.10026880502700805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,8,128,1,bfloat16,fp8,0,0.09809280037879944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,24,128,1,bfloat16,bfloat16,0,0.08651040196418762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,24,128,1,bfloat16,fp8,0,0.06853920221328735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,24,128,1,fp8,fp8,0,0.06841440200805664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,1,128,1,bfloat16,bfloat16,0,0.04619680047035217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,1,128,1,bfloat16,fp8,0,0.04578720033168793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,1,128,1,fp8,fp8,0,0.04552319943904877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,2,128,1,bfloat16,bfloat16,0,0.04840640127658844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,2,128,1,bfloat16,fp8,0,0.04667679965496063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,2,128,1,fp8,fp8,0,0.047620800137519834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,4,128,1,bfloat16,bfloat16,0,0.050507199764251706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,4,128,1,bfloat16,fp8,0,0.04948480129241943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,4,128,1,fp8,fp8,0,0.04873600006103516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,8,128,1,bfloat16,bfloat16,0,0.05663359761238098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,8,128,1,bfloat16,fp8,0,0.05272639989852905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,24,8,128,1,fp8,fp8,0,0.05262399911880493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,24,128,1,bfloat16,bfloat16,0,0.04622560143470764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,24,128,1,bfloat16,fp8,0,0.035569599270820616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,24,128,1,fp8,fp8,0,0.03617280125617981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,1,128,1,bfloat16,bfloat16,0,0.02330400049686432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,24,4,128,1,bfloat16,fp8,0,0.3325376033782959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,1,128,1,bfloat16,fp8,0,0.024296000599861145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,2,128,1,bfloat16,bfloat16,0,0.023614400625228883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,2,128,1,fp8,fp8,0,0.02436159998178482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,2,128,1,bfloat16,fp8,0,0.024561600387096406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,4,128,1,bfloat16,bfloat16,0,0.0247311994433403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,4,128,1,bfloat16,fp8,0,0.02455199956893921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,4,128,1,fp8,fp8,0,0.02481600046157837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,8,128,1,bfloat16,bfloat16,0,0.0314848005771637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,8,128,1,bfloat16,fp8,0,0.025155198574066163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,8,128,1,fp8,fp8,0,0.02547360062599182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,24,128,1,bfloat16,bfloat16,0,0.025727999210357667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,24,128,1,bfloat16,fp8,0,0.02016319930553436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,24,128,1,fp8,fp8,0,0.020695999264717102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,1,128,1,bfloat16,bfloat16,0,0.01767839938402176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,1,128,1,bfloat16,fp8,0,0.018486399948596955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,1,128,1,fp8,fp8,0,0.019036799669265747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,2,128,1,bfloat16,bfloat16,0,0.017905600368976593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,2,128,1,bfloat16,fp8,0,0.018596799671649934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,2,128,1,fp8,fp8,0,0.01899999976158142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,4,128,1,bfloat16,bfloat16,0,0.018408000469207764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,4,128,1,bfloat16,fp8,0,0.018692800402641298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,4,128,1,fp8,fp8,0,0.019232000410556793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,8,128,1,bfloat16,bfloat16,0,0.01903360038995743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,8,128,1,bfloat16,fp8,0,0.018995200097560883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,24,8,128,1,fp8,fp8,0,0.019495999813079833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,24,128,1,bfloat16,fp8,0,0.014168000221252442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,24,128,1,fp8,fp8,0,0.014558400213718414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,1,128,1,bfloat16,bfloat16,0,0.013014400005340576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,1,128,1,bfloat16,fp8,0,0.01377120018005371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,1,128,1,fp8,fp8,0,0.013593600690364837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,2,128,1,bfloat16,bfloat16,0,0.013206399977207184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,2,128,1,bfloat16,fp8,0,0.013380800187587739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,2,128,1,fp8,fp8,0,0.013663999736309052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,4,128,1,bfloat16,bfloat16,0,0.013233600556850434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,4,128,1,bfloat16,fp8,0,0.013865600526332855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,4,128,1,fp8,fp8,0,0.013612799346446991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,8,128,1,bfloat16,bfloat16,0,0.01379680037498474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,8,128,1,bfloat16,fp8,0,0.01364160031080246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,8,128,1,fp8,fp8,0,0.01385280042886734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,24,128,1,bfloat16,bfloat16,0,0.013600000739097595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,24,128,1,bfloat16,fp8,0,0.014054399728775025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,24,128,1,fp8,fp8,0,0.013617600500583648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,1,128,1,bfloat16,bfloat16,0,0.01284479945898056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,1,128,1,bfloat16,fp8,0,0.013232000172138214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,1,128,1,fp8,fp8,0,0.013607999682426453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,2,128,1,bfloat16,bfloat16,0,0.012614400684833526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,2,128,1,bfloat16,fp8,0,0.013622400164604188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,1,128,1,fp8,fp8,0,0.08618239760398864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,2,128,1,fp8,fp8,0,0.013055999577045441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,4,128,1,bfloat16,bfloat16,0,0.013350400328636169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,4,128,1,bfloat16,fp8,0,0.013451200723648072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,4,128,1,fp8,fp8,0,0.013254399597644805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,8,128,1,bfloat16,fp8,0,0.01356160044670105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,8,128,1,fp8,fp8,0,0.013339200615882873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,24,128,1,bfloat16,bfloat16,0,0.013395200669765472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,24,128,1,bfloat16,fp8,0,0.013353599607944489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,8,128,1,fp8,fp8,0,0.10022560358047486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,24,128,1,fp8,fp8,0,0.013736000657081604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,1,128,1,bfloat16,bfloat16,0,0.012276799976825714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,1,128,1,fp8,fp8,0,0.013142399489879608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,2,128,1,bfloat16,bfloat16,0,0.012299200147390365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,2,128,1,bfloat16,fp8,0,0.01316159963607788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,2,128,1,fp8,fp8,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,4,128,1,bfloat16,bfloat16,0,0.012875199317932129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,4,128,1,fp8,fp8,0,0.013254399597644805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,8,128,1,bfloat16,bfloat16,0,0.012835200130939483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,8,128,1,bfloat16,fp8,0,0.013308799266815186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,8,128,1,fp8,fp8,0,0.0133775994181633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,1,128,1,bfloat16,bfloat16,0,0.14785759449005126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,1,128,1,bfloat16,fp8,0,0.15602240562438965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,1,128,1,fp8,fp8,0,0.15582239627838135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,2,128,1,bfloat16,bfloat16,0,0.1532367944717407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,24,24,128,1,bfloat16,bfloat16,0,0.015193599462509155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,2,128,1,bfloat16,fp8,0,0.16102880239486694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,2,128,1,fp8,fp8,0,0.16034879684448242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,4,128,1,bfloat16,bfloat16,0,0.16477760076522827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,4,128,1,bfloat16,fp8,0,0.16980799436569213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,4,128,1,fp8,fp8,0,0.17486720085144042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,8,128,1,bfloat16,bfloat16,0,0.21580801010131836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,8,128,1,bfloat16,fp8,0,0.2173232078552246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,24,8,128,1,fp8,fp8,0,0.21945600509643554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,24,128,1,bfloat16,bfloat16,0,0.19129760265350343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,24,128,1,bfloat16,fp8,0,0.18262399435043336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,1,128,1,bfloat16,bfloat16,0,0.08069440126419067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,24,2,128,1,bfloat16,fp8,0,0.08760960102081299
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,1,128,1,bfloat16,fp8,0,0.0858735978603363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,24,8,128,1,bfloat16,bfloat16,0,0.013255999982357025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,24,128,1,fp8,fp8,0,0.1833807945251465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,1,128,1,fp8,fp8,0,0.08614400029182434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,2,128,1,bfloat16,fp8,0,0.08791040182113648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,2,128,1,bfloat16,bfloat16,0,0.08344799876213074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,2,128,1,fp8,fp8,0,0.08716319799423218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,4,128,1,bfloat16,fp8,0,0.09120799899101258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,4,128,1,fp8,fp8,0,0.09116640090942382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,1,128,1,bfloat16,fp8,0,0.013126400113105775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,8,128,1,bfloat16,bfloat16,0,0.11433440446853638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,8,128,1,bfloat16,fp8,0,0.1148576021194458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,8,128,1,fp8,fp8,0,0.11597119569778443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,24,128,1,bfloat16,bfloat16,0,0.10354399681091309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,1,128,1,bfloat16,bfloat16,0,0.04723840057849884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,24,128,1,bfloat16,fp8,0,0.09564319849014283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,1,128,1,fp8,fp8,0,0.046116799116134644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,1,128,1,bfloat16,fp8,0,0.045633599162101746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,2,128,1,bfloat16,bfloat16,0,0.049267199635505673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,2,128,1,bfloat16,fp8,0,0.047947201132774356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,2,128,1,fp8,fp8,0,0.04782080054283142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,4,128,1,bfloat16,bfloat16,0,0.05211520195007324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,24,1,128,1,fp8,fp8,0,0.024748800694942473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,4,128,1,bfloat16,fp8,0,0.050195199251174924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,4,128,1,fp8,fp8,0,0.05009120106697083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,8,128,1,bfloat16,bfloat16,0,0.06466879844665527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,8,128,1,bfloat16,fp8,0,0.061382400989532473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,8,128,1,fp8,fp8,0,0.06144800186157227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,24,128,1,bfloat16,fp8,0,0.048835200071334836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,24,128,1,bfloat16,bfloat16,0,0.05651199817657471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,24,128,1,fp8,fp8,0,0.04883039891719818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,1,128,1,bfloat16,bfloat16,0,0.023217600584030152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,1,128,1,bfloat16,fp8,0,0.025067201256752013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,1,128,1,fp8,fp8,0,0.024806399643421174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,2,128,1,bfloat16,bfloat16,0,0.0239439994096756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,2,128,1,bfloat16,fp8,0,0.024860799312591553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,2,128,1,fp8,fp8,0,0.025044798851013184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,4,128,1,bfloat16,bfloat16,0,0.02529599964618683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,4,128,1,bfloat16,fp8,0,0.025334399938583375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,4,128,1,fp8,fp8,0,0.02515200078487396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,8,128,1,bfloat16,bfloat16,0,0.034673601388931274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,8,128,1,fp8,fp8,0,0.030287998914718627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,24,128,1,bfloat16,bfloat16,0,0.029764801263809204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,24,128,1,bfloat16,fp8,0,0.02450399994850159
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,24,128,1,fp8,fp8,0,0.024872000515460967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,1,128,1,bfloat16,fp8,0,0.0187376007437706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,1,128,1,fp8,fp8,0,0.01894879937171936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,2,128,1,bfloat16,bfloat16,0,0.017718400061130523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,2,128,1,bfloat16,fp8,0,0.018908800184726716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,2,128,1,fp8,fp8,0,0.018862399458885192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,4,128,1,bfloat16,bfloat16,0,0.018012799322605133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,4,128,1,bfloat16,fp8,0,0.018963199853897095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,4,128,1,fp8,fp8,0,0.019070400297641753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,8,128,1,bfloat16,bfloat16,0,0.018220800161361694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,8,128,1,bfloat16,fp8,0,0.01926400065422058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,8,128,1,fp8,fp8,0,0.01905120015144348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,24,128,1,bfloat16,bfloat16,0,0.01817920058965683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,24,128,1,bfloat16,fp8,0,0.01855359971523285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,24,128,1,fp8,fp8,0,0.01875839978456497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,1,128,1,bfloat16,bfloat16,0,0.01305440068244934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,1,128,1,bfloat16,fp8,0,0.013691200315952301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,1,128,1,fp8,fp8,0,0.013707199692726135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,2,128,1,bfloat16,bfloat16,0,0.013278399407863618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,2,128,1,bfloat16,fp8,0,0.013819199800491334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,2,128,1,fp8,fp8,0,0.013711999356746673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,4,128,1,bfloat16,bfloat16,0,0.013281600177288055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,4,128,1,bfloat16,fp8,0,0.013761599361896516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,4,128,1,fp8,fp8,0,0.014017599821090698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,8,128,1,bfloat16,bfloat16,0,0.013512000441551208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,8,128,1,bfloat16,fp8,0,0.013947199285030364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,24,8,128,1,fp8,fp8,0,0.013886399567127228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,24,128,1,bfloat16,bfloat16,0,0.013131199777126313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,24,128,1,bfloat16,fp8,0,0.013663999736309052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,24,128,1,fp8,fp8,0,0.013540799915790557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,1,128,1,bfloat16,bfloat16,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,1,128,1,bfloat16,fp8,0,0.013255999982357025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,1,128,1,fp8,fp8,0,0.013516800105571746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,2,128,1,bfloat16,bfloat16,0,0.012705600261688233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,24,4,128,1,bfloat16,bfloat16,0,0.0906768023967743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,2,128,1,bfloat16,fp8,0,0.013500800728797913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,2,128,1,fp8,fp8,0,0.013264000415802002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,4,128,1,bfloat16,bfloat16,0,0.013067199289798737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,4,128,1,bfloat16,fp8,0,0.013355199992656708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,4,128,1,fp8,fp8,0,0.013308799266815186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,8,128,1,bfloat16,bfloat16,0,0.01289760023355484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,8,128,1,bfloat16,fp8,0,0.013382400572299957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,24,8,128,1,fp8,fp8,0,0.013390399515628815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,24,128,1,bfloat16,bfloat16,0,0.012751999497413635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,24,24,128,1,fp8,fp8,0,0.09549919962882995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,24,128,1,fp8,fp8,0,0.01329279989004135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,1,128,1,bfloat16,bfloat16,0,0.012455999851226807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,1,128,1,bfloat16,fp8,0,0.013179199397563934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,1,128,1,fp8,fp8,0,0.013036799430847169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,2,128,1,bfloat16,bfloat16,0,0.012511999905109405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,2,128,1,bfloat16,fp8,0,0.013148799538612366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,2,128,1,fp8,fp8,0,0.0131632000207901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,4,128,1,bfloat16,bfloat16,0,0.01257600039243698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,4,128,1,bfloat16,fp8,0,0.01316480040550232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,4,128,1,fp8,fp8,0,0.013129599392414093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,8,128,1,bfloat16,bfloat16,0,0.012668800354003907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,8,128,1,bfloat16,fp8,0,0.013305599987506866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,8,128,1,fp8,fp8,0,0.013070400059223174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,24,128,1,bfloat16,bfloat16,0,0.012638400495052337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,24,128,1,bfloat16,fp8,0,0.01329440027475357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,24,128,1,fp8,fp8,0,0.013284799456596375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,1,128,1,bfloat16,bfloat16,0,0.0122079998254776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,1,128,1,bfloat16,fp8,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,1,128,1,fp8,fp8,0,0.012772800028324127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,2,128,1,bfloat16,bfloat16,0,0.012332800030708312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,2,128,1,bfloat16,fp8,0,0.012758399546146392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,2,128,1,fp8,fp8,0,0.012787200510501862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,4,128,1,bfloat16,bfloat16,0,0.012307199835777282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,4,128,1,bfloat16,fp8,0,0.012783999741077422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,4,128,1,fp8,fp8,0,0.012615999579429627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,8,128,1,bfloat16,bfloat16,0,0.012278400361537933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,8,128,1,bfloat16,fp8,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,24,8,128,1,fp8,fp8,0,0.012936000525951386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,1,128,1,bfloat16,bfloat16,0,0.081740802526474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,1,128,1,bfloat16,fp8,0,0.08686400055885315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,24,8,128,1,bfloat16,fp8,0,0.03006559908390045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,24,1,128,1,bfloat16,bfloat16,0,0.01777600049972534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,1,128,1,fp8,fp8,0,0.08717439770698547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,2,128,1,bfloat16,bfloat16,0,0.08335679769515991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,2,128,1,bfloat16,fp8,0,0.08859840035438538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,4,128,1,bfloat16,bfloat16,0,0.10439200401306152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,4,128,1,bfloat16,fp8,0,0.10729759931564331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,4,128,1,fp8,fp8,0,0.10740799903869629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,8,128,1,bfloat16,bfloat16,0,0.11601439714431763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,8,128,1,bfloat16,fp8,0,0.11730400323867798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,8,128,1,fp8,fp8,0,0.11674400568008422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,24,128,1,bfloat16,bfloat16,0,0.14669280052185057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,24,128,1,bfloat16,fp8,0,0.14869279861450196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,24,128,1,fp8,fp8,0,0.14796639680862428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,1,128,1,bfloat16,bfloat16,0,0.04823200106620788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,1,128,1,bfloat16,fp8,0,0.04649600088596344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,1,128,1,fp8,fp8,0,0.04710879921913147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,2,128,1,bfloat16,bfloat16,0,0.049876800179481505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,2,128,1,bfloat16,fp8,0,0.04884800016880035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,2,128,1,fp8,fp8,0,0.04880639910697937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,4,128,1,bfloat16,bfloat16,0,0.05941920280456543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,4,128,1,bfloat16,fp8,0,0.059033602476119995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,4,128,1,fp8,fp8,0,0.05909759998321533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,8,128,1,bfloat16,bfloat16,0,0.065665602684021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,8,128,1,bfloat16,fp8,0,0.06217120289802551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,24,8,128,1,fp8,fp8,0,0.06288639903068542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,24,128,1,bfloat16,bfloat16,0,0.07895200252532959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,1,128,1,bfloat16,bfloat16,0,0.023844799399375914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,24,128,1,fp8,fp8,0,0.07686880230903625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,24,128,1,bfloat16,fp8,0,0.0773855984210968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,1,128,1,bfloat16,fp8,0,0.025310400128364562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,1,128,1,fp8,fp8,0,0.025276800990104674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,2,128,1,bfloat16,bfloat16,0,0.024158400297164918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,2,128,1,bfloat16,fp8,0,0.02552480101585388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,2,128,1,fp8,fp8,0,0.025248000025749208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,4,128,1,bfloat16,bfloat16,0,0.029691201448440552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,4,128,1,fp8,fp8,0,0.029867199063301087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,8,128,1,bfloat16,bfloat16,0,0.03533119857311249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,8,128,1,fp8,fp8,0,0.030527999997138976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,24,128,1,bfloat16,bfloat16,0,0.04082080125808716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,24,24,128,1,bfloat16,fp8,0,0.013513599336147309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,24,128,1,bfloat16,fp8,0,0.0385343998670578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,24,128,1,fp8,fp8,0,0.03856160044670105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,1,128,1,bfloat16,bfloat16,0,0.01793919950723648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,1,128,1,bfloat16,fp8,0,0.019366399943828584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,1,128,1,fp8,fp8,0,0.019233599305152893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,2,128,1,bfloat16,bfloat16,0,0.018092800676822663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,2,128,1,fp8,fp8,0,0.019283199310302736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,4,128,1,bfloat16,bfloat16,0,0.01793760061264038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,4,128,1,bfloat16,fp8,0,0.01926400065422058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,4,128,1,fp8,fp8,0,0.019147199392318726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,8,128,1,bfloat16,bfloat16,0,0.018849599361419677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,8,128,1,bfloat16,fp8,0,0.01931679993867874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,8,128,1,fp8,fp8,0,0.019280000030994414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,24,128,1,bfloat16,bfloat16,0,0.02239679992198944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,24,128,1,bfloat16,fp8,0,0.02298399955034256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,24,128,1,fp8,fp8,0,0.02350880056619644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,1,128,1,bfloat16,bfloat16,0,0.013040000200271606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,1,128,1,bfloat16,fp8,0,0.013956800103187561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,1,128,1,fp8,fp8,0,0.013795199990272521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,2,128,1,bfloat16,bfloat16,0,0.013502399623394012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,2,128,1,bfloat16,fp8,0,0.013736000657081604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,2,128,1,fp8,fp8,0,0.013859200477600097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,4,128,1,bfloat16,bfloat16,0,0.012984000146389008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,4,128,1,bfloat16,fp8,0,0.013940800726413728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,4,128,1,fp8,fp8,0,0.013630400598049163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,8,128,1,bfloat16,bfloat16,0,0.013483199477195739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,8,128,1,bfloat16,fp8,0,0.013710400462150574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,24,8,128,1,fp8,fp8,0,0.014214399456977844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,24,2,128,1,fp8,fp8,0,0.08771039843559265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,24,128,1,bfloat16,fp8,0,0.018142400681972502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,1,128,1,bfloat16,bfloat16,0,0.01295360028743744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,1,128,1,bfloat16,fp8,0,0.013603200018405915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,1,128,1,fp8,fp8,0,0.013683199882507324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,2,128,1,bfloat16,bfloat16,0,0.013027200102806091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,2,128,1,bfloat16,fp8,0,0.013601599633693695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,2,128,1,fp8,fp8,0,0.013519999384880067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,4,128,1,bfloat16,bfloat16,0,0.012943999469280243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,4,128,1,bfloat16,fp8,0,0.013518400490283966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,4,128,1,fp8,fp8,0,0.013523200154304504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,8,128,1,bfloat16,bfloat16,0,0.012945599853992462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,8,128,1,bfloat16,fp8,0,0.013700799643993377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,8,128,1,fp8,fp8,0,0.013556799292564392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,24,128,1,bfloat16,bfloat16,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,24,128,1,bfloat16,fp8,0,0.013592000305652618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,24,128,1,fp8,fp8,0,0.013572800159454345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,4,128,1,bfloat16,fp8,0,0.02959200143814087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,1,128,1,bfloat16,bfloat16,0,0.012494400143623352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,24,8,128,1,bfloat16,fp8,0,0.031092798709869383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,1,128,1,fp8,fp8,0,0.013143999874591828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,1,128,1,bfloat16,fp8,0,0.013219200074672699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,2,128,1,bfloat16,bfloat16,0,0.0125231996178627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,24,4,128,1,bfloat16,fp8,0,0.012884800136089326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,2,128,1,bfloat16,fp8,0,0.013155199587345123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,2,128,1,fp8,fp8,0,0.013198399543762207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,4,128,1,bfloat16,bfloat16,0,0.012476799637079239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,4,128,1,bfloat16,fp8,0,0.013232000172138214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,4,128,1,fp8,fp8,0,0.013198399543762207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,8,128,1,bfloat16,bfloat16,0,0.012508800625801087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,8,128,1,bfloat16,fp8,0,0.013065600395202636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,24,8,128,1,fp8,fp8,0,0.01313920021057129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,24,2,128,1,bfloat16,fp8,0,0.01924159973859787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,24,128,1,bfloat16,fp8,0,0.013347199559211731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,24,128,1,fp8,fp8,0,0.01342719942331314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,1,128,1,bfloat16,bfloat16,0,0.012172800302505494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,1,128,1,bfloat16,fp8,0,0.012849600613117218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,1,128,1,fp8,fp8,0,0.012783999741077422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,2,128,1,bfloat16,bfloat16,0,0.012225600332021714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,2,128,1,bfloat16,fp8,0,0.012894399464130402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,2,128,1,fp8,fp8,0,0.012803199887275695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,4,128,1,bfloat16,bfloat16,0,0.012363199889659882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,4,128,1,bfloat16,fp8,0,0.012863999605178833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,4,128,1,fp8,fp8,0,0.01290079951286316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,8,128,1,bfloat16,bfloat16,0,0.012308800220489502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,8,128,1,bfloat16,fp8,0,0.012595200538635254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,8,128,1,fp8,fp8,0,0.012803199887275695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,24,128,1,bfloat16,bfloat16,0,0.012486399710178375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,24,128,1,bfloat16,fp8,0,0.013070400059223174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,24,128,1,fp8,fp8,0,0.013313600420951843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,1,128,1,bfloat16,bfloat16,0,0.012118399888277055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,1,128,1,bfloat16,fp8,0,0.01268800050020218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,2,128,1,bfloat16,bfloat16,0,0.0122079998254776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,2,128,1,bfloat16,fp8,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,2,128,1,fp8,fp8,0,0.012582400441169738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,4,128,1,bfloat16,bfloat16,0,0.012211199849843979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,4,128,1,bfloat16,fp8,0,0.01271360069513321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,4,128,1,fp8,fp8,0,0.01260959953069687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,8,128,1,bfloat16,bfloat16,0,0.012017600238323212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,8,128,1,bfloat16,fp8,0,0.012665599584579468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,8,128,1,fp8,fp8,0,0.012572799623012543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,1,128,1,bfloat16,bfloat16,0,0.04946880042552948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,1,128,1,bfloat16,fp8,0,0.048974400758743285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,1,128,1,fp8,fp8,0,0.0483711987733841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,2,128,1,bfloat16,bfloat16,0,0.058336001634597776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,2,128,1,bfloat16,fp8,0,0.058379197120666505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,2,128,1,fp8,fp8,0,0.05796639919281006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,24,128,1,bfloat16,bfloat16,0,0.01717280000448227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,4,128,1,bfloat16,bfloat16,0,0.06186400055885315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,4,128,1,bfloat16,fp8,0,0.060836797952651976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,24,24,128,1,fp8,fp8,0,0.018089599907398224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,8,128,1,bfloat16,bfloat16,0,0.09473119974136353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,8,128,1,bfloat16,fp8,0,0.0996496021747589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,8,128,1,fp8,fp8,0,0.09977440237998962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,24,128,1,bfloat16,bfloat16,0,0.12506719827651977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,1,128,1,bfloat16,bfloat16,0,0.024566400051116943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,24,128,1,fp8,fp8,0,0.13422720432281493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,1,128,1,bfloat16,fp8,0,0.02598400115966797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,1,128,1,fp8,fp8,0,0.026422399282455444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,2,128,1,bfloat16,bfloat16,0,0.028003200888633728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,2,128,1,fp8,fp8,0,0.030755200982093812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,4,128,1,bfloat16,bfloat16,0,0.029844799637794496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,4,128,1,bfloat16,fp8,0,0.031004801392555237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,4,128,1,fp8,fp8,0,0.03112959861755371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,8,128,1,bfloat16,bfloat16,0,0.051286399364471436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,8,128,1,fp8,fp8,0,0.04919680058956146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,24,128,1,bfloat16,bfloat16,0,0.0639519989490509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,24,128,1,bfloat16,fp8,0,0.06628000140190124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,24,128,1,fp8,fp8,0,0.06632320284843445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,1,128,1,bfloat16,bfloat16,0,0.01836639940738678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,1,128,1,bfloat16,fp8,0,0.019912000000476836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,1,128,1,fp8,fp8,0,0.01988479942083359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,2,128,1,bfloat16,bfloat16,0,0.018348799645900728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,2,128,1,bfloat16,fp8,0,0.019912000000476836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,24,24,128,1,bfloat16,bfloat16,0,0.012828800082206725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,2,128,1,fp8,fp8,0,0.020019200444221497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,4,128,1,bfloat16,bfloat16,0,0.018356800079345703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,4,128,1,bfloat16,fp8,0,0.019944000244140624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,4,128,1,fp8,fp8,0,0.019687999784946442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,8,128,1,bfloat16,bfloat16,0,0.026260799169540404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,8,128,1,bfloat16,fp8,0,0.028492799401283263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,24,8,128,1,fp8,fp8,0,0.028748801350593566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,24,128,1,bfloat16,bfloat16,0,0.03367680013179779
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,24,128,1,bfloat16,fp8,0,0.037339198589324954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,24,128,1,fp8,fp8,0,0.03723039925098419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,24,1,128,1,fp8,fp8,0,0.012753599882125854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,1,128,1,bfloat16,bfloat16,0,0.013390399515628815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,1,128,1,bfloat16,fp8,0,0.01427839994430542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,1,128,1,fp8,fp8,0,0.014395199716091156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,2,128,1,bfloat16,fp8,0,0.01438080072402954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,2,128,1,fp8,fp8,0,0.014340800046920777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,4,128,1,bfloat16,bfloat16,0,0.013937599956989288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,4,128,1,bfloat16,fp8,0,0.014238399267196656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,4,128,1,fp8,fp8,0,0.01449120044708252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,8,128,1,bfloat16,bfloat16,0,0.017478400468826295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,8,128,1,bfloat16,fp8,0,0.018620799481868743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,8,128,1,fp8,fp8,0,0.01860480010509491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,24,128,1,bfloat16,bfloat16,0,0.02099999934434891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,24,128,1,bfloat16,fp8,0,0.022867199778556824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,24,128,1,fp8,fp8,0,0.02269600033760071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,24,4,128,1,fp8,fp8,0,0.06067519783973694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,1,128,1,bfloat16,bfloat16,0,0.012985600531101227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,1,128,1,bfloat16,fp8,0,0.013663999736309052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,1,128,1,fp8,fp8,0,0.013753600418567657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,2,128,1,bfloat16,fp8,0,0.013391999900341034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,2,128,1,fp8,fp8,0,0.013628800213336945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,24,128,1,bfloat16,fp8,0,0.13544000387191774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,4,128,1,bfloat16,bfloat16,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,4,128,1,bfloat16,fp8,0,0.013391999900341034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,4,128,1,fp8,fp8,0,0.013451200723648072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,8,128,1,bfloat16,bfloat16,0,0.012856000661849975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,8,128,1,bfloat16,fp8,0,0.013495999574661254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,2,128,1,bfloat16,fp8,0,0.030451199412345885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,8,128,1,fp8,fp8,0,0.013518400490283966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,24,128,1,bfloat16,bfloat16,0,0.01658399999141693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,24,128,1,bfloat16,fp8,0,0.01802240014076233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,24,128,1,fp8,fp8,0,0.017705599963665008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,1,128,1,bfloat16,bfloat16,0,0.012529599666595458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,1,128,1,bfloat16,fp8,0,0.013331200182437896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,1,128,1,fp8,fp8,0,0.013289600610733032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,24,8,128,1,bfloat16,fp8,0,0.049384000897407535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,2,128,1,bfloat16,fp8,0,0.013396799564361572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,2,128,1,fp8,fp8,0,0.013128000497817992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,4,128,1,bfloat16,bfloat16,0,0.012385600060224534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,4,128,1,fp8,fp8,0,0.01310880035161972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,8,128,1,bfloat16,bfloat16,0,0.012531200051307678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,8,128,1,bfloat16,fp8,0,0.013079999387264252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,8,128,1,fp8,fp8,0,0.013187199831008911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,24,128,1,bfloat16,fp8,0,0.01340160071849823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,24,128,1,bfloat16,bfloat16,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,24,128,1,fp8,fp8,0,0.013299199938774108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,1,128,1,bfloat16,bfloat16,0,0.01212640032172203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,1,128,1,bfloat16,fp8,0,0.012883199751377106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,1,128,1,fp8,fp8,0,0.012758399546146392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,2,128,1,bfloat16,bfloat16,0,0.012115199863910676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,2,128,1,bfloat16,fp8,0,0.012899200618267059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,2,128,1,fp8,fp8,0,0.01279039978981018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,4,128,1,bfloat16,bfloat16,0,0.012092799693346024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,4,128,1,bfloat16,fp8,0,0.01287200003862381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,4,128,1,fp8,fp8,0,0.01276639997959137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,8,128,1,bfloat16,fp8,0,0.012833599746227265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,8,128,1,fp8,fp8,0,0.012964800000190735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,24,128,1,bfloat16,bfloat16,0,0.012596799433231354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,24,128,1,bfloat16,fp8,0,0.01334560066461563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,24,128,1,fp8,fp8,0,0.013467200100421906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,1,128,1,bfloat16,bfloat16,0,0.012153600156307221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,1,128,1,bfloat16,fp8,0,0.012839999794960023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,1,128,1,fp8,fp8,0,0.012779200077056884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,2,128,1,bfloat16,bfloat16,0,0.012438400089740754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,2,128,1,bfloat16,fp8,0,0.013187199831008911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,2,128,1,fp8,fp8,0,0.012758399546146392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,4,128,1,bfloat16,bfloat16,0,0.011939200013875962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,4,128,1,bfloat16,fp8,0,0.012590399384498597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,4,128,1,fp8,fp8,0,0.012582400441169738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,8,128,1,bfloat16,bfloat16,0,0.01175519973039627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,8,128,1,bfloat16,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,24,8,128,1,fp8,fp8,0,0.012569600343704223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,24,128,1,bfloat16,bfloat16,0,0.012380799651145935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,24,128,1,bfloat16,fp8,0,0.01324319988489151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,24,128,1,fp8,fp8,0,0.013176000118255616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,1,128,1,bfloat16,bfloat16,0,0.011964800208806992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,1,128,1,bfloat16,fp8,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,1,128,1,fp8,fp8,0,0.012625600397586822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,2,128,1,bfloat16,bfloat16,0,0.011956799775362015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,2,128,1,bfloat16,fp8,0,0.012758399546146392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,24,2,128,1,bfloat16,bfloat16,0,0.013363200426101684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,2,128,1,fp8,fp8,0,0.012720000743865967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,4,128,1,bfloat16,fp8,0,0.012585599720478059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,4,128,1,fp8,fp8,0,0.01266079992055893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,8,128,1,bfloat16,bfloat16,0,0.01186240017414093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,8,128,1,bfloat16,fp8,0,0.012630400061607362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,8,128,1,fp8,fp8,0,0.012438400089740754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,1,128,1,bfloat16,bfloat16,0,0.02088800072669983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,1,128,1,bfloat16,fp8,0,0.022940799593925476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,1,128,1,fp8,fp8,0,0.022473600506782532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,2,128,1,bfloat16,bfloat16,0,0.028519999980926514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,2,128,1,bfloat16,fp8,0,0.03107680082321167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,2,128,1,fp8,fp8,0,0.031891199946403506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,4,128,1,bfloat16,bfloat16,0,0.04275839924812317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,4,128,1,bfloat16,fp8,0,0.049291199445724486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,4,128,1,fp8,fp8,0,0.04878720045089722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,8,128,1,bfloat16,bfloat16,0,0.07109119892120361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,8,128,1,bfloat16,fp8,0,0.08411359786987305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,24,8,128,1,fp8,fp8,0,0.08462079763412475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,24,128,1,bfloat16,bfloat16,0,0.09770399928092957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,24,128,1,bfloat16,fp8,0,0.11936960220336915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,1,128,1,bfloat16,bfloat16,0,0.014735999703407287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,24,128,1,fp8,fp8,0,0.11929600238800049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,2,128,1,bfloat16,bfloat16,0,0.012441600114107132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,1,128,1,bfloat16,fp8,0,0.015436799824237823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,1,128,1,fp8,fp8,0,0.015702399611473083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,2,128,1,bfloat16,bfloat16,0,0.01839359998703003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,2,128,1,fp8,fp8,0,0.019964799284934998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,4,128,1,bfloat16,bfloat16,0,0.025897601246833803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,4,128,1,bfloat16,fp8,0,0.028892800211906433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,4,128,1,fp8,fp8,0,0.028996801376342772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,8,128,1,bfloat16,bfloat16,0,0.04021599888801575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,8,128,1,bfloat16,fp8,0,0.04658559858798981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,8,128,1,fp8,fp8,0,0.046500799059867856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,24,128,1,bfloat16,bfloat16,0,0.05472800135612488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,24,128,1,bfloat16,fp8,0,0.06404160261154175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,24,8,128,1,bfloat16,bfloat16,0,0.012004800140857697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,24,128,1,fp8,fp8,0,0.06415200233459473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,1,128,1,bfloat16,fp8,0,0.014209599792957306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,1,128,1,fp8,fp8,0,0.013911999762058258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,2,128,1,bfloat16,bfloat16,0,0.013659200072288514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,2,128,1,bfloat16,fp8,0,0.014393599331378936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,2,128,1,fp8,fp8,0,0.014550399780273438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,4,128,1,bfloat16,bfloat16,0,0.01738400012254715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,4,128,1,bfloat16,fp8,0,0.01881439983844757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,4,128,1,fp8,fp8,0,0.01881919950246811
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,8,128,1,bfloat16,bfloat16,0,0.024884800612926482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,8,128,1,bfloat16,fp8,0,0.027665600180625916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,8,128,1,fp8,fp8,0,0.027699199318885804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,24,128,1,bfloat16,bfloat16,0,0.032260799407958986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,24,128,1,bfloat16,fp8,0,0.03656319975852966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,24,4,128,1,bfloat16,bfloat16,0,0.011913599818944931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,24,128,1,fp8,fp8,0,0.036545601487159726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,1,128,1,bfloat16,bfloat16,0,0.012716799974441528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,1,128,1,bfloat16,fp8,0,0.013145600259304047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,1,128,1,fp8,fp8,0,0.013355199992656708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,2,128,1,bfloat16,bfloat16,0,0.012622399628162384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,2,128,1,bfloat16,fp8,0,0.013595199584960938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,2,128,1,fp8,fp8,0,0.013299199938774108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,4,128,1,bfloat16,bfloat16,0,0.012916800379753113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,4,128,1,bfloat16,fp8,0,0.013580800592899322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,4,128,1,fp8,fp8,0,0.013683199882507324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,8,128,1,bfloat16,bfloat16,0,0.016627199947834015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,8,128,1,bfloat16,fp8,0,0.01812160015106201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,24,8,128,1,fp8,fp8,0,0.018054400384426118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,24,128,1,bfloat16,bfloat16,0,0.02027200013399124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,24,2,128,1,bfloat16,bfloat16,0,0.012651200592517852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,24,128,1,bfloat16,fp8,0,0.022193600237369538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,24,128,1,fp8,fp8,0,0.022414399683475493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,1,128,1,bfloat16,bfloat16,0,0.011771199852228164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,1,128,1,bfloat16,fp8,0,0.01252480000257492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,1,128,1,fp8,fp8,0,0.01279519945383072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,2,128,1,bfloat16,fp8,0,0.012639999389648438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,2,128,1,fp8,fp8,0,0.012910400331020356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,4,128,1,bfloat16,bfloat16,0,0.01207199990749359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,4,128,1,bfloat16,fp8,0,0.013121600449085235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,4,128,1,fp8,fp8,0,0.012833599746227265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,8,128,1,bfloat16,bfloat16,0,0.012198399752378464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,8,128,1,bfloat16,fp8,0,0.01290079951286316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,8,128,1,fp8,fp8,0,0.013065600395202636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,24,128,1,bfloat16,bfloat16,0,0.016182400286197662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,24,128,1,bfloat16,fp8,0,0.01765599995851517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,24,128,1,fp8,fp8,0,0.01762239933013916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,1,128,1,bfloat16,fp8,0,0.012624000012874604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,1,128,1,fp8,fp8,0,0.012558400630950928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,2,128,1,bfloat16,bfloat16,0,0.011840000003576278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,24,2,128,1,bfloat16,fp8,0,0.019937600195407867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,2,128,1,bfloat16,fp8,0,0.012774400413036346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,2,128,1,fp8,fp8,0,0.012627199292182922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,4,128,1,bfloat16,bfloat16,0,0.012153600156307221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,4,128,1,bfloat16,fp8,0,0.012892800569534301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,4,128,1,fp8,fp8,0,0.012836800515651703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,8,128,1,bfloat16,bfloat16,0,0.012283200025558471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,8,128,1,bfloat16,fp8,0,0.013009600341320038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,8,128,1,fp8,fp8,0,0.013121600449085235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,24,128,1,bfloat16,bfloat16,0,0.012412799894809723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,24,128,1,bfloat16,fp8,0,0.01342879980802536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,24,128,1,fp8,fp8,0,0.013374400138854981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,1,128,1,bfloat16,fp8,0,0.012556800246238708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,1,128,1,fp8,fp8,0,0.012516799569129943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,2,128,1,bfloat16,bfloat16,0,0.011641599982976914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,2,128,1,bfloat16,fp8,0,0.012484800070524216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,2,128,1,fp8,fp8,0,0.012747199833393097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,24,1,128,1,bfloat16,bfloat16,0,0.013273599743843078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,4,128,1,bfloat16,bfloat16,0,0.011641599982976914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,4,128,1,bfloat16,fp8,0,0.012488000094890594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,4,128,1,fp8,fp8,0,0.012390399724245072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,8,128,1,bfloat16,bfloat16,0,0.011921600252389909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,8,128,1,bfloat16,fp8,0,0.012715199589729309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,8,128,1,fp8,fp8,0,0.01266079992055893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,24,128,1,bfloat16,bfloat16,0,0.012415999919176102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,24,128,1,bfloat16,fp8,0,0.01313440054655075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,24,128,1,fp8,fp8,0,0.013265599310398103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,1,128,1,bfloat16,fp8,0,0.012337599694728852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,1,128,1,fp8,fp8,0,0.012379200011491776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,2,128,1,bfloat16,bfloat16,0,0.011499200016260147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,2,128,1,bfloat16,fp8,0,0.012280000001192093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,2,128,1,fp8,fp8,0,0.012280000001192093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,4,128,1,bfloat16,bfloat16,0,0.011785600334405899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,4,128,1,bfloat16,fp8,0,0.012439999729394913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,4,128,1,fp8,fp8,0,0.012409599870443344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,8,128,1,bfloat16,bfloat16,0,0.011827199906110763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,8,128,1,bfloat16,fp8,0,0.01247360035777092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,8,128,1,fp8,fp8,0,0.012515200674533844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,24,128,1,bfloat16,bfloat16,0,0.012225600332021714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,24,128,1,bfloat16,fp8,0,0.012988799810409546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,24,128,1,fp8,fp8,0,0.012921600043773651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,1,128,1,bfloat16,bfloat16,0,0.011457599699497223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,1,128,1,bfloat16,fp8,0,0.012217599898576736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,1,128,1,fp8,fp8,0,0.012305600196123123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,2,128,1,bfloat16,bfloat16,0,0.011472000181674958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,2,128,1,bfloat16,fp8,0,0.012331199645996094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,2,128,1,fp8,fp8,0,0.012243200093507767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,4,128,1,bfloat16,bfloat16,0,0.011681599915027619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,4,128,1,bfloat16,fp8,0,0.01239520013332367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,4,128,1,fp8,fp8,0,0.012352000176906585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,8,128,1,bfloat16,bfloat16,0,0.011670400202274323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,8,128,1,bfloat16,fp8,0,0.01239520013332367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,24,8,128,1,fp8,fp8,0,0.012211199849843979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,24,2,128,1,bfloat16,bfloat16,0,0.011852800101041793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,24,1,128,1,bfloat16,bfloat16,0,0.011553599685430526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,24,4,128,1,bfloat16,fp8,0,0.013318400084972381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,24,1,128,1,bfloat16,bfloat16,0,0.011737599968910217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,24,1,128,1,bfloat16,bfloat16,0,0.011580800265073776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,1,128,1,bfloat16,fp8,0,12.327130889892578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,1,128,1,fp8,fp8,0,12.289527893066406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,2,128,1,bfloat16,fp8,0,12.445510101318359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,2,128,1,fp8,fp8,0,12.34647216796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,1,128,1,bfloat16,bfloat16,0,14.574105834960937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,2,128,1,bfloat16,bfloat16,0,14.768804931640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,4,128,1,bfloat16,bfloat16,0,14.93321533203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,4,128,1,bfloat16,fp8,0,12.709918212890624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,16,128,1,bfloat16,bfloat16,0,7.602454376220703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,16,128,1,bfloat16,fp8,0,6.424273681640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,16,128,1,fp8,fp8,0,6.422518157958985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,4,128,1,fp8,fp8,0,12.444747161865234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,8,128,1,bfloat16,fp8,0,12.664545440673828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,8,128,1,fp8,fp8,0,12.472100830078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,1,128,1,bfloat16,bfloat16,0,7.623079681396485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,16,8,128,1,bfloat16,bfloat16,0,15.126969909667968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,1,128,1,bfloat16,fp8,0,6.197043228149414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,1,128,1,fp8,fp8,0,6.157772827148437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,2,128,1,bfloat16,fp8,0,6.25938720703125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,2,128,1,bfloat16,bfloat16,0,7.617524719238281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,2,128,1,fp8,fp8,0,6.532936096191406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,4,128,1,bfloat16,bfloat16,0,7.85081787109375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,4,128,1,bfloat16,fp8,0,6.379363250732422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,16,128,1,bfloat16,bfloat16,0,3.8008399963378907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,4,128,1,fp8,fp8,0,6.464895629882813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,8,128,1,bfloat16,fp8,0,7.043910217285156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,8,128,1,bfloat16,bfloat16,0,8.959508514404297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,16,128,1,bfloat16,fp8,0,3.745619201660156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,1,128,1,bfloat16,bfloat16,0,3.619686508178711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,16,8,128,1,fp8,fp8,0,7.219944000244141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,16,128,1,fp8,fp8,0,3.813409423828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,1,128,1,fp8,fp8,0,3.4849071502685547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,1,128,1,bfloat16,fp8,0,3.5089134216308593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,2,128,1,bfloat16,bfloat16,0,4.039519882202148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,2,128,1,bfloat16,fp8,0,3.358135986328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,2,128,1,fp8,fp8,0,3.4877647399902343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,4,128,1,bfloat16,fp8,0,3.358567810058594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,4,128,1,fp8,fp8,0,3.392435073852539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,8,128,1,bfloat16,fp8,0,3.503055953979492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,16,128,1,bfloat16,fp8,0,1.7211679458618163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,16,128,1,bfloat16,bfloat16,0,1.8210704803466797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,8,128,1,bfloat16,bfloat16,0,3.9264495849609373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,8,128,1,fp8,fp8,0,3.433095932006836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,16,128,1,fp8,fp8,0,1.655196762084961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,1,128,1,bfloat16,bfloat16,0,1.7745983123779296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,1,128,1,bfloat16,fp8,0,1.6279455184936524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,1,128,1,fp8,fp8,0,1.5585568428039551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,2,128,1,bfloat16,bfloat16,0,1.8818336486816407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,2,128,1,bfloat16,fp8,0,1.7624687194824218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,2,128,1,fp8,fp8,0,1.5502016067504882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,4,128,1,bfloat16,bfloat16,0,1.7889328002929688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,4,128,1,bfloat16,fp8,0,1.8038015365600586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,4,128,1,fp8,fp8,0,1.5781855583190918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,8,128,1,bfloat16,bfloat16,0,1.8787551879882813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,16,4,128,1,bfloat16,bfloat16,0,4.355867385864258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,8,128,1,fp8,fp8,0,1.5867792129516602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,16,8,128,1,bfloat16,fp8,0,1.8836095809936524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,1,128,1,fp8,fp8,0,7.0183868408203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,1,128,1,bfloat16,fp8,0,7.218968200683594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,2,128,1,fp8,fp8,0,7.181243133544922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,2,128,1,bfloat16,fp8,0,7.237254333496094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,1,128,1,bfloat16,bfloat16,0,8.7185791015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,2,128,1,bfloat16,bfloat16,0,8.825286102294921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,4,128,1,bfloat16,bfloat16,0,8.584193420410156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,4,128,1,bfloat16,fp8,0,7.084008026123047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,16,128,1,bfloat16,bfloat16,0,4.466497421264648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,16,128,1,bfloat16,fp8,0,3.802227020263672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,16,128,1,fp8,fp8,0,3.7798526763916014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,4,128,1,fp8,fp8,0,7.120750427246094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,8,128,1,bfloat16,fp8,0,7.349267578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,8,128,1,fp8,fp8,0,7.8034515380859375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,1,128,1,bfloat16,bfloat16,0,4.132076644897461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,16,8,128,1,bfloat16,bfloat16,0,8.977833557128907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,1,128,1,bfloat16,fp8,0,3.6241134643554687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,2,128,1,bfloat16,bfloat16,0,4.172903823852539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,2,128,1,bfloat16,fp8,0,3.5287406921386717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,2,128,1,fp8,fp8,0,3.5762096405029298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,4,128,1,bfloat16,fp8,0,3.7396865844726563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,4,128,1,bfloat16,bfloat16,0,4.309668731689453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,8,128,1,bfloat16,bfloat16,0,4.3676910400390625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,8,128,1,bfloat16,fp8,0,3.754667282104492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,16,128,1,bfloat16,bfloat16,0,2.215727996826172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,8,128,1,fp8,fp8,0,3.5926639556884767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,16,128,1,bfloat16,fp8,0,1.8397039413452148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,16,128,1,fp8,fp8,0,1.9644975662231445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,1,128,1,bfloat16,fp8,0,1.756608009338379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,1,128,1,bfloat16,bfloat16,0,2.1171695709228517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,1,128,1,fp8,fp8,0,1.9254335403442382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,4,128,1,fp8,fp8,0,3.6496849060058594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,2,128,1,bfloat16,fp8,0,1.7656976699829101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,2,128,1,bfloat16,bfloat16,0,2.0545759201049805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,16,1,128,1,fp8,fp8,0,3.5387855529785157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,2,128,1,fp8,fp8,0,1.7791248321533204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,4,128,1,bfloat16,bfloat16,0,1.9297391891479492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,16,128,1,bfloat16,bfloat16,0,1.0781375885009765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,4,128,1,bfloat16,fp8,0,2.036952018737793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,16,128,1,bfloat16,fp8,0,0.9407039642333984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,8,128,1,bfloat16,fp8,0,1.7922639846801758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,8,128,1,bfloat16,bfloat16,0,2.0711551666259767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,8,128,1,fp8,fp8,0,2.0965343475341798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,16,128,1,fp8,fp8,0,1.010484790802002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,1,128,1,bfloat16,bfloat16,0,0.9745311737060547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,1,128,1,bfloat16,fp8,0,0.9231247901916504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,1,128,1,fp8,fp8,0,0.8854592323303223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,2,128,1,bfloat16,bfloat16,0,1.0853072166442872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,2,128,1,bfloat16,fp8,0,0.9810144424438476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,2,128,1,fp8,fp8,0,1.0562159538269043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,4,128,1,bfloat16,fp8,0,0.9209744453430175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,4,128,1,bfloat16,bfloat16,0,1.0560111999511719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,4,128,1,fp8,fp8,0,0.901580810546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,8,128,1,bfloat16,fp8,0,1.066327953338623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,8,128,1,fp8,fp8,0,0.9922623634338379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,16,4,128,1,fp8,fp8,0,1.7867887496948243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,16,8,128,1,bfloat16,bfloat16,0,1.126534366607666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,1,128,1,bfloat16,fp8,0,5.012635040283203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,1,128,1,fp8,fp8,0,5.03233757019043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,2,128,1,bfloat16,fp8,0,5.037484741210937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,1,128,1,bfloat16,bfloat16,0,6.33422737121582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,2,128,1,fp8,fp8,0,5.049932861328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,2,128,1,bfloat16,bfloat16,0,6.2941230773925785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,16,128,1,bfloat16,bfloat16,0,3.224496078491211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,16,128,1,bfloat16,fp8,0,2.8484624862670898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,4,128,1,fp8,fp8,0,4.982128143310547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,4,128,1,bfloat16,fp8,0,5.011790466308594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,4,128,1,bfloat16,bfloat16,0,6.136735916137695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,8,128,1,fp8,fp8,0,5.102092742919922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,8,128,1,bfloat16,fp8,0,5.481407928466797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,16,8,128,1,bfloat16,bfloat16,0,6.381478500366211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,1,128,1,bfloat16,fp8,0,2.4755216598510743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,1,128,1,fp8,fp8,0,2.473734474182129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,1,128,1,bfloat16,bfloat16,0,2.872336006164551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,16,128,1,fp8,fp8,0,2.908888053894043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,2,128,1,bfloat16,bfloat16,0,2.931654357910156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,2,128,1,bfloat16,fp8,0,2.479772758483887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,2,128,1,fp8,fp8,0,2.601192092895508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,4,128,1,bfloat16,bfloat16,0,2.9648719787597657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,4,128,1,fp8,fp8,0,2.4970672607421873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,8,128,1,bfloat16,fp8,0,2.558681678771973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,16,128,1,bfloat16,bfloat16,0,1.5050095558166503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,16,128,1,bfloat16,fp8,0,1.4447440147399901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,8,128,1,bfloat16,bfloat16,0,3.0655616760253905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,8,128,1,fp8,fp8,0,2.635537528991699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,1,128,1,bfloat16,bfloat16,0,1.394472026824951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,16,128,1,fp8,fp8,0,1.4386816024780273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,1,128,1,bfloat16,fp8,0,1.259000015258789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,1,128,1,fp8,fp8,0,1.4266608238220215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,2,128,1,bfloat16,fp8,0,1.439236831665039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,2,128,1,bfloat16,bfloat16,0,1.6652032852172851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,2,128,1,fp8,fp8,0,1.2491264343261719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,4,128,1,bfloat16,fp8,0,1.2887951850891113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,4,128,1,fp8,fp8,0,1.2563952445983886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,16,128,1,bfloat16,bfloat16,0,0.7910912036895752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,4,128,1,bfloat16,bfloat16,0,1.5456303596496581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,8,128,1,bfloat16,bfloat16,0,1.4684384346008301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,8,128,1,bfloat16,fp8,0,1.3327152252197265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,16,128,1,bfloat16,fp8,0,0.6823935985565186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,16,4,128,1,bfloat16,fp8,0,2.5278032302856444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,16,128,1,fp8,fp8,0,0.6816448211669922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,16,8,128,1,fp8,fp8,0,1.535587215423584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,1,128,1,bfloat16,bfloat16,0,0.7516320228576661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,1,128,1,bfloat16,fp8,0,0.7355472087860108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,1,128,1,fp8,fp8,0,0.7111792087554931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,2,128,1,bfloat16,bfloat16,0,0.6812160015106201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,2,128,1,bfloat16,fp8,0,0.6265727996826171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,2,128,1,fp8,fp8,0,0.6282368183135987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,4,128,1,bfloat16,bfloat16,0,0.7058191776275635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,4,128,1,bfloat16,fp8,0,0.7187583923339844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,4,128,1,fp8,fp8,0,0.7343232154846191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,8,128,1,bfloat16,bfloat16,0,0.7608191967010498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,8,128,1,bfloat16,fp8,0,0.6879776000976563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,16,8,128,1,fp8,fp8,0,0.6675631999969482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,1,128,1,bfloat16,fp8,0,6.475838470458984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,1,128,1,fp8,fp8,0,6.69664306640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,2,128,1,bfloat16,fp8,0,6.568875122070312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,2,128,1,fp8,fp8,0,6.5678558349609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,4,128,1,bfloat16,fp8,0,6.576739501953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,1,128,1,bfloat16,bfloat16,0,7.842993927001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,2,128,1,bfloat16,bfloat16,0,8.006617736816406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,4,128,1,bfloat16,bfloat16,0,7.9935150146484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,16,128,1,bfloat16,fp8,0,3.453764724731445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,16,128,1,bfloat16,bfloat16,0,4.222097778320313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,16,128,1,fp8,fp8,0,3.6017471313476563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,1,128,1,bfloat16,bfloat16,0,3.7530303955078126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,4,128,1,fp8,fp8,0,6.538384246826172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,8,128,1,bfloat16,fp8,0,6.645779418945312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,8,128,1,fp8,fp8,0,7.499951934814453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,1,128,1,bfloat16,fp8,0,3.233047866821289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,16,8,128,1,bfloat16,bfloat16,0,8.542889404296876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,1,128,1,fp8,fp8,0,3.427324676513672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,2,128,1,bfloat16,fp8,0,3.36889762878418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,2,128,1,bfloat16,bfloat16,0,4.1483009338378904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,2,128,1,fp8,fp8,0,3.271372985839844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,4,128,1,bfloat16,bfloat16,0,4.371916961669922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,4,128,1,bfloat16,fp8,0,3.4310928344726563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,16,128,1,bfloat16,bfloat16,0,2.0531856536865236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,16,128,1,bfloat16,fp8,0,1.7225488662719726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,8,128,1,bfloat16,fp8,0,3.2940319061279295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,8,128,1,fp8,fp8,0,3.4755199432373045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,8,128,1,bfloat16,bfloat16,0,4.448419189453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,1,128,1,bfloat16,bfloat16,0,1.8076576232910155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,1,128,1,bfloat16,fp8,0,1.7510047912597657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,1,128,1,fp8,fp8,0,1.6193647384643555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,16,128,1,fp8,fp8,0,2.3864816665649413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,2,128,1,bfloat16,bfloat16,0,2.012384033203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,2,128,1,bfloat16,fp8,0,1.8305728912353516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,16,4,128,1,fp8,fp8,0,3.295614242553711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,2,128,1,fp8,fp8,0,1.8210399627685547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,4,128,1,bfloat16,bfloat16,0,2.0581872940063475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,4,128,1,bfloat16,fp8,0,1.6534719467163086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,4,128,1,fp8,fp8,0,1.6891664505004882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,16,128,1,bfloat16,bfloat16,0,0.978700828552246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,16,128,1,fp8,fp8,0,0.9003456115722657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,16,128,1,bfloat16,fp8,0,0.9940943717956543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,1,128,1,bfloat16,bfloat16,0,0.8808048248291016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,8,128,1,fp8,fp8,0,1.6571504592895507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,8,128,1,bfloat16,fp8,0,1.6850255966186523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,16,8,128,1,bfloat16,bfloat16,0,2.170155143737793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,1,128,1,bfloat16,fp8,0,0.884654426574707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,1,128,1,fp8,fp8,0,0.8348799705505371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,2,128,1,bfloat16,bfloat16,0,0.9185791969299316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,2,128,1,bfloat16,fp8,0,0.9365695953369141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,2,128,1,fp8,fp8,0,0.8138128280639648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,4,128,1,bfloat16,bfloat16,0,0.9263055801391602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,4,128,1,bfloat16,fp8,0,1.0070431709289551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,4,128,1,fp8,fp8,0,0.8724335670471192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,16,128,1,bfloat16,bfloat16,0,0.48300957679748535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,8,128,1,bfloat16,bfloat16,0,0.9634880065917969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,16,128,1,bfloat16,fp8,0,0.51419038772583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,8,128,1,bfloat16,fp8,0,0.9356464385986328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,16,128,1,fp8,fp8,0,0.5157760143280029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,16,8,128,1,fp8,fp8,0,0.9327775955200195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,1,128,1,bfloat16,bfloat16,0,0.4441728115081787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,1,128,1,bfloat16,fp8,0,0.4258863925933838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,1,128,1,fp8,fp8,0,0.4933119773864746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,2,128,1,bfloat16,fp8,0,0.4628511905670166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,2,128,1,bfloat16,bfloat16,0,0.44400959014892577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,2,128,1,fp8,fp8,0,0.444214391708374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,4,128,1,bfloat16,fp8,0,0.43728318214416506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,4,128,1,bfloat16,bfloat16,0,0.5123167991638183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,4,128,1,fp8,fp8,0,0.4351759910583496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,8,128,1,bfloat16,bfloat16,0,0.46351041793823244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,8,128,1,bfloat16,fp8,0,0.47172160148620607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,16,8,128,1,fp8,fp8,0,0.45808959007263184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,1,128,1,fp8,fp8,0,3.7587936401367186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,2,128,1,bfloat16,fp8,0,3.7562591552734377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,1,128,1,bfloat16,fp8,0,3.9131839752197264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,2,128,1,fp8,fp8,0,3.804212951660156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,1,128,1,bfloat16,bfloat16,0,4.355657577514648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,4,128,1,bfloat16,fp8,0,3.915697479248047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,2,128,1,bfloat16,bfloat16,0,4.3465007781982425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,4,128,1,bfloat16,bfloat16,0,4.626054382324218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,16,128,1,bfloat16,fp8,0,2.026628875732422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,16,128,1,fp8,fp8,0,2.0976463317871095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,16,128,1,bfloat16,bfloat16,0,2.4181791305541993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,1,128,1,bfloat16,bfloat16,0,2.264286422729492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,4,128,1,fp8,fp8,0,3.776559829711914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,8,128,1,bfloat16,fp8,0,3.989254379272461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,8,128,1,fp8,fp8,0,3.8487152099609374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,16,8,128,1,bfloat16,bfloat16,0,4.679838562011719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,1,128,1,fp8,fp8,0,1.8699424743652344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,2,128,1,bfloat16,bfloat16,0,2.152779197692871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,2,128,1,bfloat16,fp8,0,2.132481575012207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,2,128,1,fp8,fp8,0,1.885977554321289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,4,128,1,bfloat16,fp8,0,1.8880064010620117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,4,128,1,bfloat16,bfloat16,0,2.4547151565551757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,4,128,1,fp8,fp8,0,2.0709280014038085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,8,128,1,bfloat16,bfloat16,0,2.148849678039551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,16,128,1,bfloat16,bfloat16,0,1.1930255889892578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,8,128,1,bfloat16,fp8,0,2.0412464141845703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,16,128,1,bfloat16,fp8,0,1.0130687713623048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,8,128,1,fp8,fp8,0,1.932187271118164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,16,128,1,fp8,fp8,0,1.2209823608398438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,1,128,1,bfloat16,bfloat16,0,1.021673583984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,1,128,1,bfloat16,fp8,0,1.069859218597412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,1,128,1,fp8,fp8,0,1.0292464256286622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,16,1,128,1,bfloat16,fp8,0,1.9061712265014648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,2,128,1,bfloat16,fp8,0,0.9334527969360351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,2,128,1,fp8,fp8,0,0.9450624465942383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,4,128,1,bfloat16,fp8,0,1.037324810028076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,4,128,1,bfloat16,bfloat16,0,1.1055407524108887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,4,128,1,fp8,fp8,0,1.1088031768798827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,8,128,1,bfloat16,bfloat16,0,1.0980575561523438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,16,128,1,bfloat16,bfloat16,0,0.6010528087615967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,8,128,1,fp8,fp8,0,1.0050928115844726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,16,128,1,bfloat16,fp8,0,0.5419680118560791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,16,128,1,fp8,fp8,0,0.5927296161651612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,1,128,1,bfloat16,bfloat16,0,0.5273712158203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,1,128,1,bfloat16,fp8,0,0.5042175769805908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,2,128,1,bfloat16,bfloat16,0,0.5170032024383545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,1,128,1,fp8,fp8,0,0.5306191921234131
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,2,128,1,bfloat16,fp8,0,0.4966432094573975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,2,128,1,fp8,fp8,0,0.49526081085205076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,4,128,1,bfloat16,bfloat16,0,0.5348959922790527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,4,128,1,fp8,fp8,0,0.4958047866821289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,8,128,1,bfloat16,bfloat16,0,0.5562928199768067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,8,128,1,bfloat16,fp8,0,0.508296012878418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,8,128,1,fp8,fp8,0,0.4977935791015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,16,128,1,bfloat16,fp8,0,0.27354719638824465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,16,128,1,bfloat16,bfloat16,0,0.33471519947052003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,16,128,1,fp8,fp8,0,0.2820048093795776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,1,128,1,bfloat16,bfloat16,0,0.25990560054779055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,1,128,1,bfloat16,fp8,0,0.24753119945526122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,1,128,1,fp8,fp8,0,0.2676448106765747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,2,128,1,bfloat16,bfloat16,0,0.27047998905181886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,2,128,1,bfloat16,fp8,0,0.2428431987762451
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,2,128,1,fp8,fp8,0,0.24786241054534913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,4,128,1,bfloat16,bfloat16,0,0.2771935939788818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,4,128,1,bfloat16,fp8,0,0.2598655939102173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,4,128,1,fp8,fp8,0,0.24522719383239747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,2,128,1,bfloat16,bfloat16,0,1.0174495697021484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,8,128,1,bfloat16,bfloat16,0,0.28198239803314207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,8,128,1,bfloat16,fp8,0,0.26803839206695557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,16,8,128,1,fp8,fp8,0,0.26316640377044676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,16,8,128,1,bfloat16,fp8,0,0.971827220916748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,16,4,128,1,bfloat16,fp8,0,0.49625601768493655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,1,128,1,bfloat16,fp8,0,3.5466846466064452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,1,128,1,fp8,fp8,0,3.5692302703857424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,2,128,1,bfloat16,fp8,0,3.5523025512695314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,2,128,1,bfloat16,bfloat16,0,3.886705780029297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,1,128,1,bfloat16,bfloat16,0,4.292544174194336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,2,128,1,fp8,fp8,0,3.5206912994384765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,4,128,1,bfloat16,bfloat16,0,4.196387100219726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,16,128,1,bfloat16,bfloat16,0,2.285420799255371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,4,128,1,bfloat16,fp8,0,3.5725887298583983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,4,128,1,fp8,fp8,0,3.560124969482422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,16,128,1,bfloat16,fp8,0,1.9461584091186523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,8,128,1,fp8,fp8,0,3.679216003417969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,8,128,1,bfloat16,fp8,0,3.848011016845703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,16,8,128,1,bfloat16,bfloat16,0,4.453777694702149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,16,128,1,fp8,fp8,0,2.087646484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,1,128,1,bfloat16,fp8,0,1.7578512191772462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,1,128,1,fp8,fp8,0,1.7484512329101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,1,128,1,bfloat16,bfloat16,0,2.1334720611572267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,2,128,1,bfloat16,bfloat16,0,1.9451871871948243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,2,128,1,fp8,fp8,0,1.7720224380493164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,2,128,1,bfloat16,fp8,0,1.8901872634887695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,4,128,1,bfloat16,bfloat16,0,2.1005199432373045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,4,128,1,bfloat16,fp8,0,1.7870655059814453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,4,128,1,fp8,fp8,0,1.7753711700439454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,16,128,1,bfloat16,fp8,0,0.9808239936828613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,16,128,1,bfloat16,bfloat16,0,1.1297103881835937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,8,128,1,bfloat16,bfloat16,0,2.0302703857421873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,8,128,1,bfloat16,fp8,0,1.9925151824951173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,16,8,128,1,fp8,fp8,0,1.9976991653442382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,1,128,1,bfloat16,bfloat16,0,0.9434991836547851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,16,128,1,fp8,fp8,0,1.0899776458740233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,1,128,1,bfloat16,fp8,0,0.8839152336120606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,1,128,1,fp8,fp8,0,0.8861184120178223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,2,128,1,fp8,fp8,0,0.8891023635864258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,2,128,1,bfloat16,bfloat16,0,1.0178688049316407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,2,128,1,bfloat16,fp8,0,1.0196975708007812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,4,128,1,bfloat16,bfloat16,0,1.0021615982055665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,4,128,1,fp8,fp8,0,0.9693872451782226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,4,128,1,bfloat16,fp8,0,1.0280431747436523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,16,128,1,bfloat16,bfloat16,0,0.5932400226593018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,8,128,1,bfloat16,fp8,0,0.9508416175842285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,8,128,1,bfloat16,bfloat16,0,1.0361136436462401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,16,128,1,fp8,fp8,0,0.5262608051300048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,1,128,1,bfloat16,bfloat16,0,0.4947184085845947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,16,8,128,1,fp8,fp8,0,0.9260208129882812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,1,128,1,fp8,fp8,0,0.48863840103149414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,2,128,1,bfloat16,bfloat16,0,0.49088320732116697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,2,128,1,bfloat16,fp8,0,0.5101232051849365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,2,128,1,fp8,fp8,0,0.4709440231323242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,4,128,1,bfloat16,bfloat16,0,0.4928351879119873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,4,128,1,bfloat16,fp8,0,0.45633759498596194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,16,128,1,bfloat16,bfloat16,0,0.29085919857025144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,16,128,1,bfloat16,fp8,0,0.25632801055908205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,4,128,1,fp8,fp8,0,0.48773441314697263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,8,128,1,bfloat16,bfloat16,0,0.533568000793457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,8,128,1,bfloat16,fp8,0,0.5011760234832764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,8,128,1,fp8,fp8,0,0.47656641006469724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,16,128,1,fp8,fp8,0,0.25726239681243895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,1,128,1,bfloat16,bfloat16,0,0.24512639045715331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,1,128,1,bfloat16,fp8,0,0.2401952028274536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,1,128,1,fp8,fp8,0,0.24632000923156738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,2,128,1,bfloat16,bfloat16,0,0.25787200927734377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,2,128,1,bfloat16,fp8,0,0.23110558986663818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,2,128,1,fp8,fp8,0,0.23387680053710938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,4,128,1,bfloat16,bfloat16,0,0.25412640571594236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,4,128,1,bfloat16,fp8,0,0.24148159027099608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,4,128,1,fp8,fp8,0,0.24989759922027588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,8,128,1,bfloat16,bfloat16,0,0.2752608060836792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,16,128,1,bfloat16,bfloat16,0,0.15280319452285768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,16,128,1,bfloat16,fp8,0,0.13655840158462523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,8,128,1,fp8,fp8,0,0.24119040966033936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,16,128,1,fp8,fp8,0,0.14200799465179442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,1,128,1,bfloat16,bfloat16,0,0.13105119466781617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,1,128,1,bfloat16,fp8,0,0.11827199459075928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,2,128,1,bfloat16,bfloat16,0,0.1272704005241394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,2,128,1,bfloat16,fp8,0,0.12508480548858641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,4,128,1,bfloat16,fp8,0,0.11840800046920777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,4,128,1,fp8,fp8,0,0.12226400375366211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,8,128,1,bfloat16,bfloat16,0,0.13677120208740234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,8,128,1,bfloat16,fp8,0,0.12783199548721313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,8,128,1,fp8,fp8,0,0.12735199928283691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,16,128,1,bfloat16,fp8,0,0.5414656162261963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,16,1,128,1,bfloat16,fp8,0,0.4892767906188965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,1,128,1,bfloat16,bfloat16,0,2.440007972717285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,16,8,128,1,bfloat16,fp8,0,0.24105119705200195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,1,128,1,bfloat16,fp8,0,2.089313507080078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,1,128,1,fp8,fp8,0,0.11936800479888916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,1,128,1,fp8,fp8,0,2.0958015441894533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,2,128,1,fp8,fp8,0,0.1257151961326599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,16,4,128,1,bfloat16,bfloat16,0,0.13394240140914918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,2,128,1,bfloat16,fp8,0,2.0930416107177736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,2,128,1,fp8,fp8,0,2.0915807723999023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,2,128,1,bfloat16,bfloat16,0,2.574179267883301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,4,128,1,bfloat16,fp8,0,2.1294511795043944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,4,128,1,fp8,fp8,0,2.141859245300293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,4,128,1,bfloat16,bfloat16,0,2.4068576812744142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,8,128,1,bfloat16,bfloat16,0,2.5008960723876954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,16,128,1,bfloat16,bfloat16,0,1.4609503746032715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,16,128,1,fp8,fp8,0,1.2011648178100587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,1,128,1,bfloat16,bfloat16,0,1.258452796936035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,1,128,1,bfloat16,fp8,0,1.183521556854248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,8,128,1,bfloat16,fp8,0,2.202649688720703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,16,8,128,1,fp8,fp8,0,2.198137664794922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,2,128,1,bfloat16,bfloat16,0,1.1250864028930665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,2,128,1,fp8,fp8,0,1.1770463943481446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,4,128,1,bfloat16,fp8,0,1.0737759590148925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,4,128,1,bfloat16,bfloat16,0,1.22042875289917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,4,128,1,fp8,fp8,0,1.0683568000793457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,16,128,1,bfloat16,fp8,0,0.5994336128234863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,8,128,1,bfloat16,bfloat16,0,1.3171135902404785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,8,128,1,bfloat16,fp8,0,1.138047981262207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,8,128,1,fp8,fp8,0,1.1043359756469726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,16,128,1,fp8,fp8,0,0.5992608070373535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,1,128,1,bfloat16,fp8,0,0.5304656028747559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,1,128,1,bfloat16,bfloat16,0,0.6079904079437256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,16,128,1,bfloat16,fp8,0,1.2012304306030273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,1,128,1,fp8,fp8,0,0.530459213256836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,2,128,1,bfloat16,fp8,0,0.533070421218872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,2,128,1,bfloat16,bfloat16,0,0.566806411743164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,2,128,1,fp8,fp8,0,0.5892848014831543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,4,128,1,bfloat16,fp8,0,0.5384736061096191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,4,128,1,bfloat16,bfloat16,0,0.5839695930480957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,4,128,1,fp8,fp8,0,0.5406816005706787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,8,128,1,bfloat16,bfloat16,0,0.6170000076293946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,16,128,1,bfloat16,bfloat16,0,0.36155200004577637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,8,128,1,bfloat16,fp8,0,0.5874688148498535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,2,128,1,bfloat16,fp8,0,1.0731472015380858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,8,128,1,fp8,fp8,0,0.5574111938476562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,16,128,1,bfloat16,fp8,0,0.3072000026702881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,16,128,1,fp8,fp8,0,0.3138592004776001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,1,128,1,bfloat16,bfloat16,0,0.28552320003509524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,1,128,1,bfloat16,fp8,0,0.2821984052658081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,1,128,1,fp8,fp8,0,0.27224318981170653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,2,128,1,bfloat16,bfloat16,0,0.286627197265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,2,128,1,bfloat16,fp8,0,0.27314879894256594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,2,128,1,fp8,fp8,0,0.27900478839874265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,4,128,1,bfloat16,bfloat16,0,0.29903359413146974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,4,128,1,bfloat16,fp8,0,0.28692800998687745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,4,128,1,fp8,fp8,0,0.2776128053665161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,8,128,1,bfloat16,bfloat16,0,0.3114207983016968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,8,128,1,bfloat16,fp8,0,0.28566560745239256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,16,128,1,bfloat16,fp8,0,0.16368160247802735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,16,128,1,fp8,fp8,0,0.15922240018844605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,1,128,1,bfloat16,bfloat16,0,0.14660320281982422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,1,128,1,bfloat16,fp8,0,0.14026559591293336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,1,128,1,fp8,fp8,0,0.14431359767913818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,2,128,1,bfloat16,bfloat16,0,0.15014400482177734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,2,128,1,bfloat16,fp8,0,0.14442559480667114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,2,128,1,fp8,fp8,0,0.14439359903335572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,16,16,128,1,bfloat16,bfloat16,0,0.7481840133666993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,4,128,1,bfloat16,bfloat16,0,0.15717120170593263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,4,128,1,bfloat16,fp8,0,0.14425599575042725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,4,128,1,fp8,fp8,0,0.14653760194778442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,8,128,1,bfloat16,bfloat16,0,0.16385600566864014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,8,128,1,fp8,fp8,0,0.15289119482040406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,16,128,1,bfloat16,fp8,0,0.08897759914398193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,16,128,1,fp8,fp8,0,0.08897119760513306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,1,128,1,bfloat16,bfloat16,0,0.08133760094642639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,1,128,1,bfloat16,fp8,0,0.07831519842147827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,1,128,1,fp8,fp8,0,0.0780623972415924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,2,128,1,bfloat16,bfloat16,0,0.0816703975200653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,2,128,1,bfloat16,fp8,0,0.07837600111961365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,2,128,1,fp8,fp8,0,0.07800800204277039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,4,128,1,bfloat16,bfloat16,0,0.08598080277442932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,4,128,1,bfloat16,fp8,0,0.07858399748802185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,4,128,1,fp8,fp8,0,0.07841439843177796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,8,128,1,bfloat16,bfloat16,0,0.09089919924736023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,8,128,1,bfloat16,fp8,0,0.08046240210533143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,8,128,1,fp8,fp8,0,0.08094559907913208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,16,1,128,1,fp8,fp8,0,1.1233344078063965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,16,128,1,bfloat16,bfloat16,0,0.1908128023147583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,1,128,1,bfloat16,fp8,0,2.085318374633789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,1,128,1,fp8,fp8,0,2.0852640151977537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,1,128,1,bfloat16,bfloat16,0,2.225057601928711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,16,16,128,1,bfloat16,bfloat16,0,0.10129760503768921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,2,128,1,bfloat16,bfloat16,0,2.2327295303344727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,2,128,1,bfloat16,fp8,0,2.083736038208008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,2,128,1,fp8,fp8,0,2.058456039428711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,4,128,1,bfloat16,bfloat16,0,2.3624704360961912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,4,128,1,bfloat16,fp8,0,2.276462364196777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,4,128,1,fp8,fp8,0,2.1087711334228514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,16,8,128,1,fp8,fp8,0,0.2915855884552002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,16,128,1,bfloat16,fp8,0,1.2451519966125488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,8,128,1,bfloat16,bfloat16,0,2.692745590209961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,16,128,1,bfloat16,bfloat16,0,1.49486083984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,16,8,128,1,bfloat16,fp8,0,0.15285600423812867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,8,128,1,bfloat16,fp8,0,2.208403205871582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,16,8,128,1,fp8,fp8,0,2.1991872787475586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,16,128,1,fp8,fp8,0,1.2359487533569335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,1,128,1,bfloat16,bfloat16,0,1.0847887992858887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,1,128,1,fp8,fp8,0,1.0874784469604493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,1,128,1,bfloat16,fp8,0,1.1117183685302734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,2,128,1,bfloat16,bfloat16,0,1.1075584411621093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,2,128,1,bfloat16,fp8,0,1.0343711853027344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,2,128,1,fp8,fp8,0,1.0364496231079101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,4,128,1,bfloat16,bfloat16,0,1.1331567764282227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,4,128,1,bfloat16,fp8,0,1.0967679977416993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,4,128,1,fp8,fp8,0,1.0920528411865233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,16,128,1,bfloat16,bfloat16,0,0.7160783767700195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,8,128,1,bfloat16,bfloat16,0,1.2460448265075683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,8,128,1,bfloat16,fp8,0,1.1303983688354493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,16,128,1,bfloat16,fp8,0,0.6279024124145508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,16,8,128,1,fp8,fp8,0,1.1060992240905763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,16,128,1,fp8,fp8,0,0.6321199893951416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,1,128,1,bfloat16,bfloat16,0,0.5462543964385986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,1,128,1,bfloat16,fp8,0,0.5168928146362305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,1,128,1,fp8,fp8,0,0.5376448154449462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,2,128,1,bfloat16,bfloat16,0,0.5617519855499268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,2,128,1,bfloat16,fp8,0,0.5336544036865234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,2,128,1,fp8,fp8,0,0.5231584072113037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,4,128,1,bfloat16,bfloat16,0,0.5776112079620361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,4,128,1,bfloat16,fp8,0,0.5447807788848877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,4,128,1,fp8,fp8,0,0.5456160068511963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,16,128,1,bfloat16,bfloat16,0,0.36594879627227783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,16,128,1,bfloat16,fp8,0,0.3211024045944214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,8,128,1,bfloat16,fp8,0,0.5720223903656005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,8,128,1,bfloat16,bfloat16,0,0.6240384101867675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,1,128,1,bfloat16,bfloat16,0,0.2782896041870117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,16,8,128,1,fp8,fp8,0,0.5628672122955323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,1,128,1,bfloat16,fp8,0,0.27258079051971434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,16,128,1,fp8,fp8,0,0.3172751903533936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,1,128,1,fp8,fp8,0,0.2661616086959839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,2,128,1,bfloat16,bfloat16,0,0.28521440029144285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,2,128,1,bfloat16,fp8,0,0.2737296104431152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,2,128,1,fp8,fp8,0,0.2695647954940796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,4,128,1,bfloat16,bfloat16,0,0.29541759490966796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,4,128,1,bfloat16,fp8,0,0.2795056104660034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,4,128,1,fp8,fp8,0,0.28022720813751223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,8,128,1,bfloat16,bfloat16,0,0.3174783945083618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,8,128,1,bfloat16,fp8,0,0.28607680797576907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,16,128,1,bfloat16,fp8,0,0.16516159772872924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,16,8,128,1,fp8,fp8,0,0.2929696083068848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,1,128,1,bfloat16,bfloat16,0,0.14612640142440797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,1,128,1,bfloat16,fp8,0,0.14256800413131715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,16,128,1,fp8,fp8,0,0.16818560361862184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,1,128,1,fp8,fp8,0,0.13997440338134765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,2,128,1,bfloat16,bfloat16,0,0.14867839813232422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,2,128,1,bfloat16,fp8,0,0.14097440242767334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,2,128,1,fp8,fp8,0,0.1435744047164917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,4,128,1,fp8,fp8,0,0.1458832025527954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,4,128,1,bfloat16,fp8,0,0.1461151957511902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,8,128,1,bfloat16,bfloat16,0,0.16435519456863404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,8,128,1,bfloat16,fp8,0,0.15273599624633788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,16,128,1,bfloat16,bfloat16,0,0.10071840286254882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,16,128,1,bfloat16,fp8,0,0.08953120112419129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,16,128,1,fp8,fp8,0,0.09003999829292297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,1,128,1,bfloat16,bfloat16,0,0.07550719976425171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,1,128,1,bfloat16,fp8,0,0.07407680153846741
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,1,128,1,fp8,fp8,0,0.07389600276947021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,2,128,1,bfloat16,bfloat16,0,0.07978079915046692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,2,128,1,bfloat16,fp8,0,0.07415040135383606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,2,128,1,fp8,fp8,0,0.07284160256385804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,4,128,1,bfloat16,bfloat16,0,0.08263999819755555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,4,128,1,fp8,fp8,0,0.07529119849205017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,4,128,1,bfloat16,fp8,0,0.07533280253410339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,8,128,1,bfloat16,bfloat16,0,0.08837119936943054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,8,128,1,bfloat16,fp8,0,0.08017920255661011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,16,8,128,1,fp8,fp8,0,0.08077120184898376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,16,128,1,bfloat16,bfloat16,0,0.05435519814491272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,16,128,1,bfloat16,fp8,0,0.04657439887523651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,16,128,1,fp8,fp8,0,0.046224001049995425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,1,128,1,bfloat16,bfloat16,0,0.04079360067844391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,1,128,1,bfloat16,fp8,0,0.041654399037361144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,1,128,1,fp8,fp8,0,0.041703999042510986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,2,128,1,bfloat16,bfloat16,0,0.04223040044307709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,2,128,1,bfloat16,fp8,0,0.04151360094547272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,2,128,1,fp8,fp8,0,0.04059999883174896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,4,128,1,bfloat16,bfloat16,0,0.04320639967918396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,4,128,1,bfloat16,fp8,0,0.041700801253318785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,4,128,1,fp8,fp8,0,0.041860800981521604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,8,128,1,bfloat16,bfloat16,0,0.04527199864387512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,8,128,1,bfloat16,fp8,0,0.04179520010948181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,16,8,128,1,fp8,fp8,0,0.04270879924297333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,1,128,1,bfloat16,bfloat16,0,1.322766399383545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,1,128,1,fp8,fp8,0,1.3026896476745606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,2,128,1,bfloat16,bfloat16,0,1.3361359596252442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,16,128,1,bfloat16,bfloat16,0,0.1936784029006958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,4,128,1,bfloat16,bfloat16,0,0.15460799932479857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,16,8,128,1,fp8,fp8,0,0.15429279804229737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,2,128,1,fp8,fp8,0,1.3455087661743164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,4,128,1,bfloat16,bfloat16,0,1.4260895729064942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,4,128,1,bfloat16,fp8,0,1.3920816421508788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,4,128,1,fp8,fp8,0,1.3308015823364259
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,16,128,1,bfloat16,bfloat16,0,0.9288191795349121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,8,128,1,bfloat16,bfloat16,0,1.5553215980529784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,8,128,1,bfloat16,fp8,0,1.4259360313415528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,8,128,1,fp8,fp8,0,1.4185263633728027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,16,128,1,bfloat16,fp8,0,0.7991951942443848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,1,128,1,bfloat16,fp8,0,1.3015199661254884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,16,128,1,fp8,fp8,0,0.8057439804077149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,1,128,1,bfloat16,bfloat16,0,0.6867263793945313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,1,128,1,bfloat16,fp8,0,0.6599616050720215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,16,2,128,1,bfloat16,fp8,0,1.3029631614685058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,1,128,1,fp8,fp8,0,0.6464096069335937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,2,128,1,bfloat16,bfloat16,0,0.6881184101104736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,2,128,1,bfloat16,fp8,0,0.651259183883667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,2,128,1,fp8,fp8,0,0.6622911930084229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,4,128,1,bfloat16,bfloat16,0,0.7272640228271484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,4,128,1,bfloat16,fp8,0,0.679969596862793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,4,128,1,fp8,fp8,0,0.6782000064849854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,8,128,1,bfloat16,bfloat16,0,0.7711455821990967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,8,128,1,bfloat16,fp8,0,0.7231232166290283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,16,128,1,bfloat16,bfloat16,0,0.46993122100830076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,1,128,1,bfloat16,bfloat16,0,0.3379152059555054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,16,128,1,bfloat16,fp8,0,0.4152575969696045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,16,128,1,fp8,fp8,0,0.402891206741333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,1,128,1,bfloat16,fp8,0,0.3293344020843506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,16,8,128,1,fp8,fp8,0,0.7096240043640136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,1,128,1,fp8,fp8,0,0.3372607946395874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,2,128,1,bfloat16,bfloat16,0,0.3480288028717041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,2,128,1,fp8,fp8,0,0.34289278984069826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,4,128,1,bfloat16,bfloat16,0,0.359883189201355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,4,128,1,bfloat16,fp8,0,0.35179040431976316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,4,128,1,fp8,fp8,0,0.34158079624176024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,8,128,1,bfloat16,bfloat16,0,0.40007519721984863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,16,128,1,bfloat16,bfloat16,0,0.2413327932357788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,8,128,1,bfloat16,fp8,0,0.36720800399780273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,8,128,1,fp8,fp8,0,0.36777920722961427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,16,128,1,bfloat16,fp8,0,0.20969760417938232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,16,128,1,fp8,fp8,0,0.2112368106842041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,1,128,1,bfloat16,bfloat16,0,0.17713760137557982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,1,128,1,bfloat16,fp8,0,0.17120640277862548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,1,128,1,fp8,fp8,0,0.17526079416275026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,2,128,1,bfloat16,bfloat16,0,0.18101279735565184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,2,128,1,fp8,fp8,0,0.17366559505462648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,4,128,1,bfloat16,bfloat16,0,0.18966079950332643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,4,128,1,bfloat16,fp8,0,0.17665120363235473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,4,128,1,fp8,fp8,0,0.18023040294647216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,8,128,1,bfloat16,bfloat16,0,0.20642080307006835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,8,128,1,bfloat16,fp8,0,0.18847039937973023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,8,128,1,fp8,fp8,0,0.19005759954452514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,16,128,1,bfloat16,bfloat16,0,0.1272047996520996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,16,128,1,bfloat16,fp8,0,0.11368639469146728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,1,128,1,bfloat16,bfloat16,0,0.09397119879722596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,16,128,1,fp8,fp8,0,0.11351519823074341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,1,128,1,bfloat16,fp8,0,0.09217600226402282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,1,128,1,fp8,fp8,0,0.09082080125808716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,2,128,1,bfloat16,bfloat16,0,0.09698879718780518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,2,128,1,bfloat16,fp8,0,0.09259520173072815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,2,128,1,fp8,fp8,0,0.09446079730987549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,4,128,1,bfloat16,bfloat16,0,0.1008031964302063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,4,128,1,fp8,fp8,0,0.09690560102462768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,8,128,1,bfloat16,bfloat16,0,0.10782560110092163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,8,128,1,bfloat16,fp8,0,0.10244319438934327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,16,128,1,bfloat16,bfloat16,0,0.07021440267562866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,8,128,1,fp8,fp8,0,0.1011680006980896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,16,128,1,bfloat16,fp8,0,0.06069599986076355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,16,128,1,fp8,fp8,0,0.06203680038452149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,1,128,1,bfloat16,fp8,0,0.049351999163627626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,1,128,1,fp8,fp8,0,0.04921759963035584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,2,128,1,bfloat16,bfloat16,0,0.05091999769210816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,2,128,1,bfloat16,fp8,0,0.04948000013828278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,2,128,1,fp8,fp8,0,0.049369600415229795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,4,128,1,bfloat16,bfloat16,0,0.055448001623153685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,4,128,1,bfloat16,fp8,0,0.049751999974250796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,4,128,1,fp8,fp8,0,0.04941760003566742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,8,128,1,bfloat16,bfloat16,0,0.06093760132789612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,8,128,1,fp8,fp8,0,0.052830398082733154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,16,128,1,bfloat16,bfloat16,0,0.03803200125694275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,16,128,1,bfloat16,fp8,0,0.031999999284744264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,16,128,1,fp8,fp8,0,0.032395198941230774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,1,128,1,bfloat16,bfloat16,0,0.03036159873008728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,1,128,1,bfloat16,fp8,0,0.030313599109649658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,1,128,1,fp8,fp8,0,0.030368000268936157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,2,128,1,bfloat16,bfloat16,0,0.03068000078201294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,2,128,1,bfloat16,fp8,0,0.030510398745536804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,2,128,1,fp8,fp8,0,0.030484798550605773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,4,128,1,bfloat16,bfloat16,0,0.031225600838661195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,4,128,1,bfloat16,fp8,0,0.03070879876613617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,4,128,1,fp8,fp8,0,0.031031998991966247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,8,128,1,bfloat16,bfloat16,0,0.03238399922847748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,8,128,1,bfloat16,fp8,0,0.03140799999237061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,16,8,128,1,fp8,fp8,0,0.03157120048999786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,16,2,128,1,bfloat16,fp8,0,0.33518400192260744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,16,2,128,1,bfloat16,fp8,0,0.1734976053237915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,1,128,1,bfloat16,bfloat16,0,1.3460880279541017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,1,128,1,bfloat16,fp8,0,1.3380415916442872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,1,128,1,fp8,fp8,0,1.3428144454956055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,2,128,1,bfloat16,bfloat16,0,1.3836015701293944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,16,4,128,1,bfloat16,fp8,0,0.09406560063362121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,1,128,1,bfloat16,bfloat16,0,0.05031520128250122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,2,128,1,bfloat16,fp8,0,1.3639391899108886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,16,8,128,1,bfloat16,fp8,0,0.053635197877883914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,2,128,1,fp8,fp8,0,1.3941264152526855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,4,128,1,bfloat16,bfloat16,0,1.4613663673400878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,4,128,1,bfloat16,fp8,0,1.3938735961914062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,4,128,1,fp8,fp8,0,1.4167280197143555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,8,128,1,bfloat16,bfloat16,0,1.6602800369262696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,1,128,1,bfloat16,bfloat16,0,0.6868192195892334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,16,128,1,bfloat16,bfloat16,0,1.0451199531555175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,8,128,1,bfloat16,fp8,0,1.518075180053711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,16,128,1,fp8,fp8,0,0.8703439712524415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,16,128,1,bfloat16,fp8,0,0.8952128410339355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,16,8,128,1,fp8,fp8,0,1.5093711853027343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,1,128,1,bfloat16,fp8,0,0.6943344116210938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,1,128,1,fp8,fp8,0,0.6739344120025634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,2,128,1,bfloat16,bfloat16,0,0.7109519958496093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,2,128,1,fp8,fp8,0,0.6772431850433349
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,4,128,1,bfloat16,bfloat16,0,0.7555168151855469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,4,128,1,bfloat16,fp8,0,0.7065792083740234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,4,128,1,fp8,fp8,0,0.7156383991241455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,16,128,1,bfloat16,bfloat16,0,0.5305967807769776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,8,128,1,bfloat16,bfloat16,0,0.8325663566589355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,16,128,1,fp8,fp8,0,0.45123682022094724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,8,128,1,bfloat16,fp8,0,0.7765888214111328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,8,128,1,fp8,fp8,0,0.7539919853210449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,1,128,1,bfloat16,bfloat16,0,0.3510319948196411
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,1,128,1,bfloat16,fp8,0,0.3456784009933472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,1,128,1,fp8,fp8,0,0.3484352111816406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,2,128,1,bfloat16,bfloat16,0,0.3632688045501709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,2,128,1,bfloat16,fp8,0,0.34603519439697267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,2,128,1,fp8,fp8,0,0.35389280319213867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,4,128,1,bfloat16,fp8,0,0.3625936031341553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,4,128,1,bfloat16,bfloat16,0,0.38375039100646974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,4,128,1,fp8,fp8,0,0.3636768102645874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,8,128,1,bfloat16,bfloat16,0,0.421065616607666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,8,128,1,bfloat16,fp8,0,0.3913424015045166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,16,128,1,bfloat16,fp8,0,0.23127520084381104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,8,128,1,fp8,fp8,0,0.39000160694122316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,1,128,1,bfloat16,bfloat16,0,0.18281439542770386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,1,128,1,bfloat16,fp8,0,0.18076159954071044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,1,128,1,fp8,fp8,0,0.1808895945549011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,2,128,1,bfloat16,bfloat16,0,0.18893120288848878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,2,128,1,bfloat16,fp8,0,0.183241605758667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,2,128,1,fp8,fp8,0,0.1826159954071045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,4,128,1,bfloat16,fp8,0,0.18818240165710448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,4,128,1,fp8,fp8,0,0.18805919885635375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,8,128,1,bfloat16,bfloat16,0,0.22299680709838868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,8,128,1,bfloat16,fp8,0,0.2019887924194336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,8,128,1,fp8,fp8,0,0.2017807960510254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,16,128,1,bfloat16,bfloat16,0,0.14261599779129028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,16,128,1,bfloat16,fp8,0,0.12211359739303589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,16,128,1,fp8,fp8,0,0.1224943995475769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,1,128,1,bfloat16,bfloat16,0,0.09825440049171448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,1,128,1,bfloat16,fp8,0,0.09609119892120362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,1,128,1,fp8,fp8,0,0.09720479846000671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,2,128,1,bfloat16,bfloat16,0,0.10052000284194947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,2,128,1,bfloat16,fp8,0,0.09732800126075744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,2,128,1,fp8,fp8,0,0.09795039892196655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,4,128,1,bfloat16,bfloat16,0,0.10689120292663574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,16,16,128,1,bfloat16,fp8,0,0.4460303783416748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,4,128,1,bfloat16,fp8,0,0.09961919784545899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,4,128,1,fp8,fp8,0,0.100164794921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,8,128,1,bfloat16,bfloat16,0,0.11746879816055297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,8,128,1,bfloat16,fp8,0,0.10608320236206055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,16,8,128,1,fp8,fp8,0,0.10687359571456909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,16,128,1,bfloat16,bfloat16,0,0.07649440169334412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,16,128,1,bfloat16,fp8,0,0.06662880182266236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,16,128,1,fp8,fp8,0,0.06586719751358032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,1,128,1,bfloat16,bfloat16,0,0.05220639705657959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,1,128,1,bfloat16,fp8,0,0.05125920176506042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,1,128,1,fp8,fp8,0,0.05154399871826172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,2,128,1,bfloat16,bfloat16,0,0.055062401294708255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,2,128,1,bfloat16,fp8,0,0.05119839906692505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,2,128,1,fp8,fp8,0,0.05219680070877075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,4,128,1,bfloat16,fp8,0,0.05188959836959839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,4,128,1,bfloat16,bfloat16,0,0.05917919874191284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,4,128,1,fp8,fp8,0,0.05201600193977356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,8,128,1,bfloat16,bfloat16,0,0.06512799859046936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,8,128,1,bfloat16,fp8,0,0.05756959915161133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,16,8,128,1,fp8,fp8,0,0.059273600578308105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,16,128,1,bfloat16,fp8,0,0.033452799916267394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,16,128,1,fp8,fp8,0,0.033102399110794066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,1,128,1,bfloat16,bfloat16,0,0.029308798909187316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,1,128,1,bfloat16,fp8,0,0.03001280128955841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,1,128,1,fp8,fp8,0,0.029864001274108886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,2,128,1,bfloat16,bfloat16,0,0.029569599032402038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,2,128,1,bfloat16,fp8,0,0.030236798524856567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,2,128,1,fp8,fp8,0,0.029766398668289184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,4,128,1,bfloat16,bfloat16,0,0.030155199766159057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,4,128,1,bfloat16,fp8,0,0.030606400966644288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,4,128,1,fp8,fp8,0,0.03017120063304901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,8,128,1,bfloat16,bfloat16,0,0.03283840119838714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,8,128,1,bfloat16,fp8,0,0.031332799792289735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,8,128,1,fp8,fp8,0,0.031009599566459656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,16,128,1,bfloat16,bfloat16,0,0.27314560413360595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,16,128,1,bfloat16,bfloat16,0,0.02651199996471405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,16,128,1,fp8,fp8,0,0.2275360107421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,16,128,1,fp8,fp8,0,0.02609440088272095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,1,128,1,bfloat16,bfloat16,0,0.02425280064344406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,1,128,1,fp8,fp8,0,0.024859200417995452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,2,128,1,bfloat16,fp8,0,0.02489439994096756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,2,128,1,fp8,fp8,0,0.025158399343490602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,4,128,1,bfloat16,bfloat16,0,0.02468319982290268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,4,128,1,bfloat16,fp8,0,0.02510719895362854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,4,128,1,fp8,fp8,0,0.024987199902534486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,16,4,128,1,bfloat16,bfloat16,0,0.19947839975357057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,8,128,1,bfloat16,bfloat16,0,0.02542720139026642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,8,128,1,bfloat16,fp8,0,0.025481599569320678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,8,128,1,fp8,fp8,0,0.025748801231384278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,16,2,128,1,bfloat16,fp8,0,0.6800096035003662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,1,128,1,bfloat16,bfloat16,0,0.9616559982299805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,1,128,1,bfloat16,fp8,0,0.9879376411437988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,1,128,1,fp8,fp8,0,0.9743472099304199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,16,16,128,1,bfloat16,bfloat16,0,0.042024001479148865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,2,128,1,bfloat16,bfloat16,0,1.0066559791564942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,2,128,1,bfloat16,fp8,0,0.9835824012756348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,16,128,1,bfloat16,fp8,0,0.026440000534057616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,2,128,1,fp8,fp8,0,0.9838879585266114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,1,128,1,bfloat16,fp8,0,0.024643200635910033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,16,2,128,1,bfloat16,bfloat16,0,0.02411839962005615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,4,128,1,bfloat16,bfloat16,0,1.0821359634399415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,4,128,1,bfloat16,fp8,0,1.0429375648498536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,4,128,1,fp8,fp8,0,1.0279359817504883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,16,128,1,bfloat16,bfloat16,0,0.8368687629699707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,16,128,1,bfloat16,fp8,0,0.7039567947387695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,8,128,1,bfloat16,bfloat16,0,1.273198413848877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,8,128,1,fp8,fp8,0,1.1614368438720704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,16,128,1,fp8,fp8,0,0.7041664123535156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,1,128,1,bfloat16,fp8,0,0.49378080368041993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,1,128,1,fp8,fp8,0,0.4938015937805176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,2,128,1,bfloat16,bfloat16,0,0.5084911823272705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,2,128,1,bfloat16,fp8,0,0.5037600040435791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,2,128,1,fp8,fp8,0,0.503385591506958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,4,128,1,bfloat16,bfloat16,0,0.5453199863433837
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,4,128,1,bfloat16,fp8,0,0.5203248023986816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,4,128,1,fp8,fp8,0,0.5208127975463868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,16,128,1,bfloat16,bfloat16,0,0.42642722129821775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,16,128,1,bfloat16,fp8,0,0.3583888053894043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,8,128,1,bfloat16,fp8,0,0.5832335948944092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,8,128,1,fp8,fp8,0,0.5819568157196044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,16,128,1,fp8,fp8,0,0.35873119831085204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,1,128,1,bfloat16,bfloat16,0,0.24700479507446288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,1,128,1,bfloat16,fp8,0,0.25372159481048584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,1,128,1,fp8,fp8,0,0.2537503957748413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,2,128,1,bfloat16,bfloat16,0,0.2583823919296265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,2,128,1,bfloat16,fp8,0,0.2566400051116943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,2,128,1,fp8,fp8,0,0.25641119480133057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,4,128,1,bfloat16,bfloat16,0,0.28116159439086913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,4,128,1,bfloat16,fp8,0,0.2681056022644043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,4,128,1,fp8,fp8,0,0.26779999732971194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,8,128,1,bfloat16,bfloat16,0,0.32586719989776614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,8,128,1,bfloat16,fp8,0,0.2969376087188721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,16,128,1,bfloat16,fp8,0,0.1863759994506836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,16,128,1,bfloat16,bfloat16,0,0.22055199146270751
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,16,128,1,fp8,fp8,0,0.1859104037284851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,1,128,1,bfloat16,bfloat16,0,0.13049919605255128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,1,128,1,bfloat16,fp8,0,0.1335312008857727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,1,128,1,fp8,fp8,0,0.1334432005882263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,2,128,1,bfloat16,bfloat16,0,0.13610559701919556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,2,128,1,bfloat16,fp8,0,0.1361248016357422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,2,128,1,fp8,fp8,0,0.13545119762420654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,4,128,1,bfloat16,bfloat16,0,0.14661600589752197
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,4,128,1,bfloat16,fp8,0,0.14054880142211915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,4,128,1,fp8,fp8,0,0.1424239993095398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,8,128,1,bfloat16,bfloat16,0,0.17025760412216187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,8,128,1,bfloat16,fp8,0,0.1555408000946045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,16,8,128,1,bfloat16,fp8,0,1.1470848083496095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,1,128,1,bfloat16,bfloat16,0,0.4809567928314209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,16,128,1,bfloat16,fp8,0,0.09950079917907714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,16,128,1,bfloat16,bfloat16,0,0.11767840385437012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,16,128,1,fp8,fp8,0,0.09957759976387023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,1,128,1,bfloat16,fp8,0,0.07256640195846557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,1,128,1,bfloat16,bfloat16,0,0.07195199728012085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,1,128,1,fp8,fp8,0,0.07322080135345459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,2,128,1,bfloat16,fp8,0,0.07385920286178589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,2,128,1,fp8,fp8,0,0.07349759936332703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,4,128,1,bfloat16,bfloat16,0,0.07938399910926819
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,4,128,1,fp8,fp8,0,0.07666079998016358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,4,128,1,bfloat16,fp8,0,0.07692000269889832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,8,128,1,bfloat16,bfloat16,0,0.0908128023147583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,8,128,1,bfloat16,fp8,0,0.08403679728507996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,8,128,1,fp8,fp8,0,0.08427680134773255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,16,128,1,bfloat16,fp8,0,0.05624960064888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,16,128,1,fp8,fp8,0,0.05700799822807312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,1,128,1,bfloat16,bfloat16,0,0.0410863995552063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,1,128,1,bfloat16,fp8,0,0.03960480093955994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,2,128,1,bfloat16,bfloat16,0,0.043644800782203674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,2,128,1,bfloat16,fp8,0,0.040174400806427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,2,128,1,fp8,fp8,0,0.04024479985237121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,4,128,1,bfloat16,bfloat16,0,0.0463344007730484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,4,128,1,bfloat16,fp8,0,0.04181120097637177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,4,128,1,fp8,fp8,0,0.04148640036582947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,8,128,1,bfloat16,bfloat16,0,0.05278880000114441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,8,128,1,bfloat16,fp8,0,0.04680159986019135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,16,8,128,1,bfloat16,bfloat16,0,0.6488416194915771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,8,128,1,fp8,fp8,0,0.04775680005550385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,16,128,1,bfloat16,bfloat16,0,0.037699198722839354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,16,128,1,fp8,fp8,0,0.03110400140285492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,16,128,1,bfloat16,fp8,0,0.030763199925422667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,1,128,1,bfloat16,bfloat16,0,0.025097599625587462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,1,128,1,bfloat16,fp8,0,0.026284798979759216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,1,128,1,fp8,fp8,0,0.025998398661613464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,2,128,1,bfloat16,bfloat16,0,0.02602880001068115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,2,128,1,bfloat16,fp8,0,0.026092800498008727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,2,128,1,fp8,fp8,0,0.026470398902893065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,4,128,1,bfloat16,bfloat16,0,0.026510399580001832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,4,128,1,bfloat16,fp8,0,0.026587200164794923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,4,128,1,fp8,fp8,0,0.026502400636672974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,8,128,1,bfloat16,bfloat16,0,0.029264000058174134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,8,128,1,fp8,fp8,0,0.027496001124382018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,16,128,1,bfloat16,bfloat16,0,0.02027200013399124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,16,8,128,1,fp8,fp8,0,0.29753119945526124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,16,128,1,bfloat16,fp8,0,0.020478400588035583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,1,128,1,bfloat16,bfloat16,0,0.017745600640773775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,16,128,1,fp8,fp8,0,0.020614400506019592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,1,128,1,bfloat16,fp8,0,0.018935999274253844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,1,128,1,fp8,fp8,0,0.019139200448989868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,2,128,1,bfloat16,bfloat16,0,0.018028800189495087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,2,128,1,bfloat16,fp8,0,0.019179199635982514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,2,128,1,fp8,fp8,0,0.019153599441051484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,4,128,1,bfloat16,bfloat16,0,0.018415999412536622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,4,128,1,bfloat16,fp8,0,0.019198399782180787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,8,128,1,bfloat16,bfloat16,0,0.019100800156593323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,8,128,1,bfloat16,fp8,0,0.019840000569820403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,8,128,1,fp8,fp8,0,0.019696000218391418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,16,128,1,bfloat16,bfloat16,0,0.01886080056428909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,16,128,1,bfloat16,fp8,0,0.019734400510787963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,16,128,1,fp8,fp8,0,0.019630399346351624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,1,128,1,bfloat16,bfloat16,0,0.017433600127696992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,1,128,1,bfloat16,fp8,0,0.018801599740982056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,16,128,1,bfloat16,bfloat16,0,0.06527199745178222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,1,128,1,fp8,fp8,0,0.01886560022830963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,2,128,1,bfloat16,bfloat16,0,0.01765599995851517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,2,128,1,bfloat16,fp8,0,0.018838399648666383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,2,128,1,fp8,fp8,0,0.019064000248908995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,16,1,128,1,fp8,fp8,0,0.04005280137062073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,4,128,1,bfloat16,bfloat16,0,0.01791359931230545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,4,128,1,bfloat16,fp8,0,0.019148799777030944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,4,128,1,fp8,fp8,0,0.019068799912929535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,8,128,1,bfloat16,bfloat16,0,0.01814880073070526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,8,128,1,bfloat16,fp8,0,0.019446399807929993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,16,8,128,1,fp8,fp8,0,0.01934240013360977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,1,128,1,bfloat16,bfloat16,0,0.39367198944091797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,1,128,1,bfloat16,fp8,0,0.4147984027862549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,1,128,1,fp8,fp8,0,0.41138558387756347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,2,128,1,bfloat16,bfloat16,0,0.4158143997192383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,2,128,1,bfloat16,fp8,0,0.419755220413208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,2,128,1,fp8,fp8,0,0.416428804397583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,16,8,128,1,bfloat16,fp8,0,0.027463999390602113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,4,128,1,bfloat16,bfloat16,0,0.46166720390319826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,4,128,1,bfloat16,fp8,0,0.44099202156066897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,4,128,1,fp8,fp8,0,0.4382656097412109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,8,128,1,bfloat16,bfloat16,0,0.5595695972442627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,16,4,128,1,fp8,fp8,0,0.019443200528621675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,16,128,1,bfloat16,bfloat16,0,0.39126560688018797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,8,128,1,bfloat16,fp8,0,0.49843997955322267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,16,128,1,bfloat16,fp8,0,0.30727999210357665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,16,8,128,1,fp8,fp8,0,0.501801586151123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,16,8,128,1,fp8,fp8,0,0.1555791974067688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,16,128,1,fp8,fp8,0,0.3069535970687866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,1,128,1,bfloat16,bfloat16,0,0.20381760597229004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,1,128,1,bfloat16,fp8,0,0.21265599727630616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,16,2,128,1,bfloat16,bfloat16,0,0.07529919743537902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,2,128,1,bfloat16,bfloat16,0,0.21727681159973145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,2,128,1,bfloat16,fp8,0,0.21799039840698242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,2,128,1,fp8,fp8,0,0.21725759506225586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,4,128,1,bfloat16,bfloat16,0,0.2377840042114258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,4,128,1,bfloat16,fp8,0,0.22617759704589843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,4,128,1,fp8,fp8,0,0.22674560546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,8,128,1,bfloat16,bfloat16,0,0.28782079219818113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,8,128,1,bfloat16,fp8,0,0.25826880931854246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,8,128,1,fp8,fp8,0,0.25809919834136963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,16,128,1,bfloat16,bfloat16,0,0.20346879959106445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,1,128,1,bfloat16,bfloat16,0,0.11070879697799682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,16,128,1,fp8,fp8,0,0.1605039954185486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,1,128,1,bfloat16,fp8,0,0.11366560459136962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,1,128,1,fp8,fp8,0,0.11425440311431885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,2,128,1,bfloat16,bfloat16,0,0.11437280178070068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,2,128,1,bfloat16,fp8,0,0.11614240407943725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,2,128,1,fp8,fp8,0,0.11520800590515137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,4,128,1,bfloat16,bfloat16,0,0.12636959552764893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,4,128,1,bfloat16,fp8,0,0.1211840033531189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,4,128,1,fp8,fp8,0,0.1197167992591858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,8,128,1,bfloat16,bfloat16,0,0.15053600072860718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,8,128,1,bfloat16,fp8,0,0.1355664014816284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,16,128,1,bfloat16,fp8,0,0.08758559823036194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,8,128,1,fp8,fp8,0,0.13524479866027833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,16,128,1,fp8,fp8,0,0.08756160140037536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,1,128,1,bfloat16,bfloat16,0,0.06225919723510742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,1,128,1,bfloat16,fp8,0,0.06286720037460328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,1,128,1,fp8,fp8,0,0.06292160153388977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,2,128,1,bfloat16,fp8,0,0.06377599835395813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,2,128,1,bfloat16,bfloat16,0,0.06483680009841919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,2,128,1,fp8,fp8,0,0.06440160274505616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,4,128,1,bfloat16,bfloat16,0,0.070033597946167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,4,128,1,bfloat16,fp8,0,0.0663424015045166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,4,128,1,fp8,fp8,0,0.06619679927825928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,8,128,1,bfloat16,bfloat16,0,0.08290560245513916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,8,128,1,bfloat16,fp8,0,0.07491679787635804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,8,128,1,fp8,fp8,0,0.07469919919967652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,16,128,1,bfloat16,bfloat16,0,0.06122400164604187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,16,128,1,bfloat16,fp8,0,0.04965600073337555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,16,128,1,fp8,fp8,0,0.050414401292800906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,1,128,1,bfloat16,fp8,0,0.034971201419830324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,1,128,1,fp8,fp8,0,0.034334400296211244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,2,128,1,bfloat16,bfloat16,0,0.03853760063648224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,2,128,1,bfloat16,fp8,0,0.0349375993013382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,2,128,1,fp8,fp8,0,0.03432320058345795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,4,128,1,bfloat16,bfloat16,0,0.04160960018634796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,4,128,1,bfloat16,fp8,0,0.036534398794174194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,4,128,1,fp8,fp8,0,0.035815998911857605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,8,128,1,bfloat16,bfloat16,0,0.04760160148143768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,8,128,1,bfloat16,fp8,0,0.0424784004688263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,8,128,1,fp8,fp8,0,0.04221920073032379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,16,128,1,bfloat16,bfloat16,0,0.034334400296211244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,16,128,1,bfloat16,fp8,0,0.025678399205207824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,16,128,1,fp8,fp8,0,0.02529120147228241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,1,128,1,bfloat16,bfloat16,0,0.02126079946756363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,1,128,1,bfloat16,fp8,0,0.02249760031700134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,1,128,1,fp8,fp8,0,0.022308799624443054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,2,128,1,bfloat16,bfloat16,0,0.02157599925994873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,2,128,1,bfloat16,fp8,0,0.022631999850273133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,2,128,1,fp8,fp8,0,0.02234400063753128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,4,128,1,bfloat16,bfloat16,0,0.022256000339984892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,4,128,1,bfloat16,fp8,0,0.022734400629997254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,4,128,1,fp8,fp8,0,0.02247200012207031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,8,128,1,bfloat16,bfloat16,0,0.024872000515460967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,8,128,1,bfloat16,fp8,0,0.023822399973869323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,16,8,128,1,fp8,fp8,0,0.023479999601840974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,16,128,1,bfloat16,bfloat16,0,0.017390400171279907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,16,128,1,bfloat16,fp8,0,0.016808000206947327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,16,128,1,fp8,fp8,0,0.016436800360679626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,1,128,1,bfloat16,bfloat16,0,0.014905600249767304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,1,128,1,bfloat16,fp8,0,0.015356799960136414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,1,128,1,fp8,fp8,0,0.015065599977970124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,2,128,1,bfloat16,bfloat16,0,0.015163199603557586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,2,128,1,bfloat16,fp8,0,0.015409600734710694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,2,128,1,fp8,fp8,0,0.01520799994468689
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,4,128,1,bfloat16,bfloat16,0,0.015433600544929505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,4,128,1,bfloat16,fp8,0,0.015433600544929505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,4,128,1,fp8,fp8,0,0.015369600057601929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,16,1,128,1,fp8,fp8,0,0.21398720741271973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,8,128,1,bfloat16,bfloat16,0,0.016113600134849547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,8,128,1,bfloat16,fp8,0,0.016047999262809753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,16,8,128,1,fp8,fp8,0,0.015883199870586395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,16,128,1,bfloat16,bfloat16,0,0.015688000619411467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,16,128,1,bfloat16,fp8,0,0.01603199988603592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,16,128,1,fp8,fp8,0,0.01597920060157776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,1,128,1,bfloat16,fp8,0,0.015011200308799743
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,2,128,1,bfloat16,fp8,0,0.015025599300861359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,2,128,1,fp8,fp8,0,0.014985600113868713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,16,16,128,1,bfloat16,fp8,0,0.15984480381011962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,4,128,1,bfloat16,bfloat16,0,0.014667199552059173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,4,128,1,bfloat16,fp8,0,0.015219199657440185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,4,128,1,fp8,fp8,0,0.015092800557613372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,8,128,1,bfloat16,bfloat16,0,0.015087999403476715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,8,128,1,bfloat16,fp8,0,0.015380799770355225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,8,128,1,fp8,fp8,0,0.015732799470424653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,16,128,1,bfloat16,bfloat16,0,0.014812800288200378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,16,128,1,bfloat16,fp8,0,0.015644800662994385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,16,128,1,fp8,fp8,0,0.015513600409030914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,1,128,1,bfloat16,bfloat16,0,0.01467359960079193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,1,128,1,bfloat16,fp8,0,0.014924800395965577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,1,128,1,fp8,fp8,0,0.015160000324249268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,2,128,1,bfloat16,bfloat16,0,0.014123199880123139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,2,128,1,bfloat16,fp8,0,0.015150399506092071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,2,128,1,fp8,fp8,0,0.014959999918937683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,4,128,1,bfloat16,bfloat16,0,0.014564800262451171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,4,128,1,bfloat16,fp8,0,0.014945599436759948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,4,128,1,fp8,fp8,0,0.015072000026702882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,8,128,1,bfloat16,bfloat16,0,0.014547200500965118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,8,128,1,bfloat16,fp8,0,0.01544640064239502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,16,8,128,1,fp8,fp8,0,0.015316799283027649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,16,1,128,1,bfloat16,bfloat16,0,0.0350928008556366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,1,128,1,bfloat16,bfloat16,0,0.19710240364074708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,1,128,1,bfloat16,fp8,0,0.2110896110534668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,1,128,1,fp8,fp8,0,0.2087023973464966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,2,128,1,bfloat16,bfloat16,0,0.20920319557189943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,2,128,1,bfloat16,fp8,0,0.2170720100402832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,2,128,1,fp8,fp8,0,0.21573278903961182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,4,128,1,bfloat16,bfloat16,0,0.23404960632324218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,4,128,1,bfloat16,fp8,0,0.23195838928222656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,4,128,1,fp8,fp8,0,0.23138720989227296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,8,128,1,bfloat16,bfloat16,0,0.2853327989578247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,8,128,1,bfloat16,fp8,0,0.25840959548950193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,16,8,128,1,fp8,fp8,0,0.2565216064453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,16,128,1,bfloat16,fp8,0,0.16178079843521118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,1,128,1,bfloat16,bfloat16,0,0.10630240440368652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,16,128,1,fp8,fp8,0,0.16109919548034668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,1,128,1,bfloat16,bfloat16,0,0.014417600631713868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,1,128,1,fp8,fp8,0,0.01515360027551651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,1,128,1,fp8,fp8,0,0.11165119409561157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,2,128,1,bfloat16,bfloat16,0,0.1111456036567688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,2,128,1,bfloat16,fp8,0,0.11468640565872193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,2,128,1,fp8,fp8,0,0.11468640565872193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,4,128,1,bfloat16,bfloat16,0,0.12257280349731445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,4,128,1,bfloat16,fp8,0,0.12186880111694336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,4,128,1,fp8,fp8,0,0.12147680521011353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,8,128,1,bfloat16,bfloat16,0,0.15011039972305298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,8,128,1,bfloat16,fp8,0,0.1368623971939087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,8,128,1,fp8,fp8,0,0.13640480041503905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,16,128,1,bfloat16,bfloat16,0,0.11106079816818237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,16,128,1,bfloat16,fp8,0,0.08700479865074158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,16,16,128,1,bfloat16,bfloat16,0,0.10980800390243531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,16,128,1,fp8,fp8,0,0.0865119993686676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,1,128,1,bfloat16,bfloat16,0,0.060166400671005246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,1,128,1,fp8,fp8,0,0.06087520122528076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,1,128,1,bfloat16,fp8,0,0.06134080290794373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,2,128,1,bfloat16,bfloat16,0,0.06265599727630615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,2,128,1,bfloat16,fp8,0,0.06266239881515503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,2,128,1,fp8,fp8,0,0.06223359704017639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,4,128,1,bfloat16,bfloat16,0,0.06859679818153382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,4,128,1,bfloat16,fp8,0,0.06570079922676086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,4,128,1,fp8,fp8,0,0.06507520079612732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,8,128,1,bfloat16,bfloat16,0,0.081523197889328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,8,128,1,bfloat16,fp8,0,0.07285119891166687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,16,8,128,1,fp8,fp8,0,0.07285919785499573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,16,128,1,bfloat16,bfloat16,0,0.06160479784011841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,16,128,1,bfloat16,fp8,0,0.047228801250457766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,16,128,1,fp8,fp8,0,0.046911999583244324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,1,128,1,bfloat16,bfloat16,0,0.03172479867935181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,1,128,1,bfloat16,fp8,0,0.030935999751091004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,2,128,1,bfloat16,bfloat16,0,0.03514400124549866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,2,128,1,bfloat16,fp8,0,0.031068798899650574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,2,128,1,fp8,fp8,0,0.031057599186897277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,4,128,1,bfloat16,bfloat16,0,0.038841599225997926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,4,128,1,bfloat16,fp8,0,0.032543998956680295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,4,128,1,fp8,fp8,0,0.03222880065441132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,8,128,1,bfloat16,bfloat16,0,0.0458976000547409
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,8,128,1,bfloat16,fp8,0,0.03910239934921265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,8,128,1,fp8,fp8,0,0.03903680145740509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,16,128,1,bfloat16,bfloat16,0,0.03205280005931854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,16,128,1,bfloat16,fp8,0,0.022035199403762817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,16,128,1,fp8,fp8,0,0.021835200488567352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,1,128,1,bfloat16,bfloat16,0,0.018219199776649476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,1,128,1,fp8,fp8,0,0.019414399564266206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,2,128,1,bfloat16,bfloat16,0,0.01865759938955307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,2,128,1,bfloat16,fp8,0,0.019398400187492372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,2,128,1,fp8,fp8,0,0.01923999935388565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,4,128,1,bfloat16,bfloat16,0,0.01924320012331009
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,4,128,1,bfloat16,fp8,0,0.019492800533771514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,4,128,1,fp8,fp8,0,0.019592000544071196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,8,128,1,bfloat16,bfloat16,0,0.021536000072956085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,16,128,1,bfloat16,bfloat16,0,0.2072848081588745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,8,128,1,bfloat16,fp8,0,0.02016319930553436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,8,128,1,fp8,fp8,0,0.02030559927225113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,16,128,1,bfloat16,fp8,0,0.015041600167751312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,16,128,1,bfloat16,bfloat16,0,0.015744000673294067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,1,128,1,bfloat16,bfloat16,0,0.013510400056838989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,1,128,1,bfloat16,fp8,0,0.014139199256896972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,1,128,1,fp8,fp8,0,0.014100800454616546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,2,128,1,bfloat16,bfloat16,0,0.013619199395179749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,16,1,128,1,bfloat16,fp8,0,0.11133760213851929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,2,128,1,bfloat16,fp8,0,0.014103999733924866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,2,128,1,fp8,fp8,0,0.013929599523544311
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,4,128,1,bfloat16,bfloat16,0,0.013847999274730682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,4,128,1,bfloat16,fp8,0,0.014214399456977844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,4,128,1,fp8,fp8,0,0.013971200585365296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,8,128,1,bfloat16,bfloat16,0,0.014460800588130951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,8,128,1,bfloat16,fp8,0,0.014470399916172027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,8,128,1,fp8,fp8,0,0.014361600577831268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,16,128,1,bfloat16,bfloat16,0,0.01404159963130951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,16,128,1,bfloat16,fp8,0,0.01412159949541092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,16,128,1,fp8,fp8,0,0.013707199692726135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,1,128,1,bfloat16,bfloat16,0,0.012718400359153748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,1,128,1,bfloat16,fp8,0,0.013692800700664521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,1,128,1,fp8,fp8,0,0.013387200236320496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,2,128,1,bfloat16,fp8,0,0.013555200397968292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,2,128,1,fp8,fp8,0,0.013648000359535218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,4,128,1,bfloat16,bfloat16,0,0.013281600177288055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,4,128,1,bfloat16,fp8,0,0.013670399785041809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,4,128,1,fp8,fp8,0,0.013819199800491334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,8,128,1,bfloat16,bfloat16,0,0.013568000495433807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,8,128,1,bfloat16,fp8,0,0.013947199285030364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,8,128,1,fp8,fp8,0,0.0135343998670578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,16,128,1,bfloat16,bfloat16,0,0.013755199313163758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,16,128,1,bfloat16,fp8,0,0.013475200533866883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,16,128,1,fp8,fp8,0,0.013600000739097595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,1,128,1,bfloat16,bfloat16,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,1,128,1,bfloat16,fp8,0,0.013374400138854981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,1,128,1,fp8,fp8,0,0.013281600177288055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,2,128,1,bfloat16,bfloat16,0,0.013150399923324585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,2,128,1,bfloat16,fp8,0,0.013236799836158752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,2,128,1,fp8,fp8,0,0.013687999546527862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,4,128,1,bfloat16,bfloat16,0,0.012895999848842621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,4,128,1,bfloat16,fp8,0,0.01358560025691986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,4,128,1,fp8,fp8,0,0.013316799700260163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,8,128,1,bfloat16,bfloat16,0,0.013065600395202636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,8,128,1,bfloat16,fp8,0,0.013382400572299957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,16,8,128,1,fp8,fp8,0,0.013755199313163758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,16,128,1,bfloat16,bfloat16,0,0.012534399330615998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,16,128,1,bfloat16,fp8,0,0.01316000074148178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,16,128,1,fp8,fp8,0,0.012916800379753113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,1,128,1,bfloat16,bfloat16,0,0.012531200051307678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,1,128,1,bfloat16,fp8,0,0.012860800325870513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,1,128,1,fp8,fp8,0,0.012934400141239167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,2,128,1,bfloat16,bfloat16,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,2,128,1,bfloat16,fp8,0,0.012956799566745758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,2,128,1,fp8,fp8,0,0.01284479945898056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,4,128,1,bfloat16,bfloat16,0,0.012577599287033081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,4,128,1,bfloat16,fp8,0,0.012939199805259705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,4,128,1,fp8,fp8,0,0.012912000715732574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,8,128,1,bfloat16,bfloat16,0,0.012641599774360657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,16,1,128,1,bfloat16,fp8,0,0.019351999461650848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,8,128,1,bfloat16,fp8,0,0.012984000146389008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,16,8,128,1,fp8,fp8,0,0.013131199777126313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,1,128,1,bfloat16,bfloat16,0,0.10655200481414795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,1,128,1,bfloat16,fp8,0,0.11216959953308106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,1,128,1,fp8,fp8,0,0.1116752028465271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,2,128,1,bfloat16,bfloat16,0,0.1128159999847412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,16,16,128,1,fp8,fp8,0,0.015124799311161041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,2,128,1,bfloat16,fp8,0,0.11435680389404297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,2,128,1,fp8,fp8,0,0.11463359594345093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,4,128,1,bfloat16,fp8,0,0.12157119512557983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,4,128,1,bfloat16,bfloat16,0,0.12641279697418212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,4,128,1,fp8,fp8,0,0.12245440483093262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,8,128,1,bfloat16,bfloat16,0,0.15170719623565673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,8,128,1,bfloat16,fp8,0,0.13711040019989013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,16,8,128,1,fp8,fp8,0,0.13622239828109742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,16,128,1,bfloat16,bfloat16,0,0.13290560245513916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,16,128,1,bfloat16,fp8,0,0.1241152048110962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,16,128,1,fp8,fp8,0,0.12408479452133178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,16,2,128,1,bfloat16,bfloat16,0,0.012963199615478515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,1,128,1,bfloat16,bfloat16,0,0.06017600297927857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,1,128,1,bfloat16,fp8,0,0.06217120289802551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,1,128,1,fp8,fp8,0,0.061862397193908694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,2,128,1,bfloat16,bfloat16,0,0.06292799711227418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,2,128,1,bfloat16,fp8,0,0.0629423975944519
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,2,128,1,fp8,fp8,0,0.06329759955406189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,4,128,1,bfloat16,bfloat16,0,0.06863679885864257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,4,128,1,bfloat16,fp8,0,0.06587039828300476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,4,128,1,fp8,fp8,0,0.06591039896011353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,8,128,1,bfloat16,fp8,0,0.0727504014968872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,16,128,1,bfloat16,bfloat16,0,0.07256479859352112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,16,128,1,fp8,fp8,0,0.06446560025215149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,1,128,1,bfloat16,bfloat16,0,0.032155200839042664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,1,128,1,bfloat16,fp8,0,0.03131200075149536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,1,128,1,fp8,fp8,0,0.03102880120277405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,2,128,1,bfloat16,bfloat16,0,0.03648959994316101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,2,128,1,bfloat16,fp8,0,0.031067198514938353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,2,128,1,fp8,fp8,0,0.03140000104904175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,4,128,1,bfloat16,bfloat16,0,0.039801600575447085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,4,128,1,bfloat16,fp8,0,0.03281440138816834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,4,128,1,fp8,fp8,0,0.03283199965953827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,8,128,1,bfloat16,bfloat16,0,0.04614560008049011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,16,1,128,1,fp8,fp8,0,0.0306768000125885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,8,128,1,fp8,fp8,0,0.039657598733901976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,16,128,1,bfloat16,bfloat16,0,0.03978399932384491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,16,128,1,bfloat16,fp8,0,0.03096640110015869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,16,128,1,fp8,fp8,0,0.0312032014131546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,1,128,1,bfloat16,bfloat16,0,0.01897120028734207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,1,128,1,bfloat16,fp8,0,0.019897599518299103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,1,128,1,fp8,fp8,0,0.019651199877262115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,2,128,1,bfloat16,bfloat16,0,0.01920959949493408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,2,128,1,bfloat16,fp8,0,0.019801600277423857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,2,128,1,fp8,fp8,0,0.019991999864578246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,4,128,1,bfloat16,bfloat16,0,0.01964319944381714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,4,128,1,bfloat16,fp8,0,0.02022880017757416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,4,128,1,fp8,fp8,0,0.019968000054359437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,8,128,1,bfloat16,bfloat16,0,0.02203039973974228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,8,128,1,bfloat16,fp8,0,0.020491200685501098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,16,8,128,1,fp8,fp8,0,0.02073120027780533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,16,128,1,bfloat16,bfloat16,0,0.018984000384807586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,16,128,1,bfloat16,fp8,0,0.019289599359035493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,16,128,1,fp8,fp8,0,0.019011199474334717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,1,128,1,bfloat16,fp8,0,0.013972799479961395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,1,128,1,fp8,fp8,0,0.014094400405883788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,16,2,128,1,bfloat16,bfloat16,0,0.01464959979057312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,2,128,1,bfloat16,bfloat16,0,0.013715200126171112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,2,128,1,bfloat16,fp8,0,0.01472959965467453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,2,128,1,fp8,fp8,0,0.014211200177669525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,4,128,1,bfloat16,bfloat16,0,0.01401599943637848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,4,128,1,bfloat16,fp8,0,0.014259199798107147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,4,128,1,fp8,fp8,0,0.0144896000623703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,8,128,1,bfloat16,bfloat16,0,0.014553600549697876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,8,128,1,bfloat16,fp8,0,0.014608000218868256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,8,128,1,fp8,fp8,0,0.01443839967250824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,16,128,1,bfloat16,bfloat16,0,0.013684800267219544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,16,128,1,bfloat16,fp8,0,0.013651199638843536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,16,128,1,fp8,fp8,0,0.01388320028781891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,1,128,1,bfloat16,bfloat16,0,0.012891200184822083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,1,128,1,bfloat16,fp8,0,0.013515199720859527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,1,128,1,fp8,fp8,0,0.013606399297714233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,2,128,1,bfloat16,bfloat16,0,0.01302880048751831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,2,128,1,bfloat16,fp8,0,0.013655999302864074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,2,128,1,fp8,fp8,0,0.01361600011587143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,4,128,1,bfloat16,bfloat16,0,0.01308639943599701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,8,128,1,bfloat16,bfloat16,0,0.08278719782829284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,4,128,1,bfloat16,fp8,0,0.013680000603199006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,4,128,1,fp8,fp8,0,0.013476799428462981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,16,128,1,bfloat16,fp8,0,0.0643231987953186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,8,128,1,bfloat16,fp8,0,0.01387999951839447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,8,128,1,bfloat16,bfloat16,0,0.01356479972600937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,16,128,1,bfloat16,bfloat16,0,0.012887999415397644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,16,128,1,bfloat16,fp8,0,0.013447999954223633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,16,128,1,fp8,fp8,0,0.013361600041389466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,1,128,1,bfloat16,bfloat16,0,0.012619200348854064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,1,128,1,bfloat16,fp8,0,0.013435199856758118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,2,128,1,bfloat16,bfloat16,0,0.012628799676895142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,1,128,1,fp8,fp8,0,0.013710400462150574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,2,128,1,bfloat16,fp8,0,0.013171200454235078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,2,128,1,fp8,fp8,0,0.013406400382518769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,4,128,1,bfloat16,bfloat16,0,0.01311040073633194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,4,128,1,bfloat16,fp8,0,0.01321599930524826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,4,128,1,fp8,fp8,0,0.013491199910640716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,8,128,1,bfloat16,bfloat16,0,0.013049599528312684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,8,128,1,bfloat16,fp8,0,0.013393600285053254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,16,128,1,bfloat16,bfloat16,0,0.012542399764060973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,16,128,1,bfloat16,fp8,0,0.012907199561595917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,16,128,1,fp8,fp8,0,0.01308639943599701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,1,128,1,bfloat16,bfloat16,0,0.012316799908876418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,1,128,1,bfloat16,fp8,0,0.012848000228404998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,1,128,1,fp8,fp8,0,0.012916800379753113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,2,128,1,bfloat16,bfloat16,0,0.012203200161457062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,2,128,1,bfloat16,fp8,0,0.012912000715732574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,2,128,1,fp8,fp8,0,0.012759999930858612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,4,128,1,bfloat16,bfloat16,0,0.012464000284671784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,4,128,1,bfloat16,fp8,0,0.01281599998474121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,4,128,1,fp8,fp8,0,0.0127920001745224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,8,128,1,bfloat16,bfloat16,0,0.012624000012874604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,8,128,1,bfloat16,fp8,0,0.01300320029258728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,16,8,128,1,fp8,fp8,0,0.013036799430847169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,16,128,1,bfloat16,bfloat16,0,0.012116800248622894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,16,128,1,bfloat16,fp8,0,0.01279039978981018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,16,128,1,fp8,fp8,0,0.012863999605178833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,1,128,1,bfloat16,bfloat16,0,0.012324800342321396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,1,128,1,bfloat16,fp8,0,0.012726399302482604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,1,128,1,fp8,fp8,0,0.012919999659061432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,2,128,1,bfloat16,bfloat16,0,0.012134400010108948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,2,128,1,bfloat16,fp8,0,0.012902399897575379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,2,128,1,fp8,fp8,0,0.01271039992570877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,4,128,1,bfloat16,bfloat16,0,0.01220960021018982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,4,128,1,bfloat16,fp8,0,0.01276959925889969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,4,128,1,fp8,fp8,0,0.01284160017967224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,8,128,1,bfloat16,bfloat16,0,0.01231359988451004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,8,128,1,bfloat16,fp8,0,0.012809599936008453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,16,1,128,1,bfloat16,bfloat16,0,0.013648000359535218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,16,8,128,1,fp8,fp8,0,0.012785600125789642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,1,128,1,bfloat16,bfloat16,0,0.06127679944038391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,1,128,1,bfloat16,fp8,0,0.06353440284729003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,1,128,1,fp8,fp8,0,0.06357439756393432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,2,128,1,bfloat16,bfloat16,0,0.06416000127792358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,2,128,1,bfloat16,fp8,0,0.0650991976261139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,2,128,1,fp8,fp8,0,0.06447039842605591
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,4,128,1,bfloat16,bfloat16,0,0.07043359875679016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,4,128,1,bfloat16,fp8,0,0.0675055980682373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,4,128,1,fp8,fp8,0,0.06755200028419495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,8,128,1,bfloat16,bfloat16,0,0.111080002784729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,8,128,1,bfloat16,fp8,0,0.10841439962387085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,16,8,128,1,fp8,fp8,0,0.10807679891586304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,16,128,1,bfloat16,bfloat16,0,0.10067520141601563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,16,128,1,bfloat16,fp8,0,0.10044959783554078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,16,128,1,fp8,fp8,0,0.10050879716873169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,1,128,1,bfloat16,bfloat16,0,0.031435200572013856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,1,128,1,bfloat16,fp8,0,0.032076799869537355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,1,128,1,fp8,fp8,0,0.03186399936676025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,2,128,1,bfloat16,bfloat16,0,0.03670080006122589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,2,128,1,bfloat16,fp8,0,0.03236800134181976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,2,128,1,fp8,fp8,0,0.03188160061836243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,4,128,1,bfloat16,bfloat16,0,0.03988640010356903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,4,128,1,bfloat16,fp8,0,0.03446879982948303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,4,128,1,fp8,fp8,0,0.0337119996547699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,16,8,128,1,fp8,fp8,0,0.07279360294342041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,8,128,1,bfloat16,bfloat16,0,0.06035040020942688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,8,128,1,bfloat16,fp8,0,0.05772160291671753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,16,8,128,1,fp8,fp8,0,0.057627201080322266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,16,128,1,bfloat16,bfloat16,0,0.05494719743728638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,16,8,128,1,fp8,fp8,0,0.013681599497795105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,16,128,1,fp8,fp8,0,0.04942240118980408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,1,128,1,bfloat16,bfloat16,0,0.018857599794864656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,1,128,1,bfloat16,fp8,0,0.020265600085258482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,1,128,1,fp8,fp8,0,0.020604799687862396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,2,128,1,bfloat16,bfloat16,0,0.019631999731063842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,2,128,1,bfloat16,fp8,0,0.020235200226306916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,2,128,1,fp8,fp8,0,0.02040639966726303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,4,128,1,bfloat16,bfloat16,0,0.02011840045452118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,4,128,1,bfloat16,fp8,0,0.020776000618934632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,4,128,1,fp8,fp8,0,0.020414400100708007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,8,128,1,bfloat16,bfloat16,0,0.028527998924255372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,8,128,1,bfloat16,fp8,0,0.029625600576400755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,8,128,1,fp8,fp8,0,0.029662400484085083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,16,128,1,bfloat16,bfloat16,0,0.027014398574829103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,16,128,1,bfloat16,fp8,0,0.02853280007839203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,16,8,128,1,fp8,fp8,0,0.013470399379730224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,1,128,1,bfloat16,bfloat16,0,0.01406240016222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,1,128,1,bfloat16,fp8,0,0.014678399264812469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,2,128,1,bfloat16,bfloat16,0,0.013891200721263885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,16,8,128,1,bfloat16,fp8,0,0.03942880034446716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,2,128,1,bfloat16,fp8,0,0.014769600331783294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,2,128,1,fp8,fp8,0,0.014657600224018097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,4,128,1,bfloat16,bfloat16,0,0.014107200503349304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,4,128,1,fp8,fp8,0,0.014912000298500061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,4,128,1,bfloat16,fp8,0,0.014619199931621552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,8,128,1,bfloat16,bfloat16,0,0.0181536003947258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,8,128,1,bfloat16,fp8,0,0.018907199800014495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,8,128,1,fp8,fp8,0,0.019172799587249757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,16,128,1,bfloat16,bfloat16,0,0.017324799299240114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,16,128,1,bfloat16,fp8,0,0.0181551992893219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,1,128,1,bfloat16,bfloat16,0,0.013115200400352477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,1,128,1,bfloat16,fp8,0,0.013446399569511413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,1,128,1,fp8,fp8,0,0.013548800349235534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,2,128,1,bfloat16,bfloat16,0,0.013225600123405457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,2,128,1,bfloat16,fp8,0,0.013532799482345582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,2,128,1,fp8,fp8,0,0.013702400028705597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,4,128,1,bfloat16,bfloat16,0,0.01313440054655075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,4,128,1,bfloat16,fp8,0,0.013777600228786468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,4,128,1,fp8,fp8,0,0.013814400136470794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,8,128,1,bfloat16,bfloat16,0,0.012939199805259705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,16,128,1,bfloat16,bfloat16,0,0.01313920021057129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,16,128,1,bfloat16,fp8,0,0.01361439973115921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,16,128,1,fp8,fp8,0,0.013540799915790557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,1,128,1,bfloat16,bfloat16,0,0.013121600449085235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,1,128,1,bfloat16,fp8,0,0.013524800539016724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,1,128,1,fp8,fp8,0,0.013740800321102142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,2,128,1,bfloat16,bfloat16,0,0.01308639943599701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,2,128,1,bfloat16,fp8,0,0.013662399351596832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,2,128,1,fp8,fp8,0,0.013324800133705138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,4,128,1,bfloat16,bfloat16,0,0.013196800649166108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,4,128,1,bfloat16,fp8,0,0.013504000008106231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,4,128,1,fp8,fp8,0,0.013465599715709686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,8,128,1,bfloat16,bfloat16,0,0.012692800164222718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,8,128,1,bfloat16,fp8,0,0.013353599607944489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,16,8,128,1,fp8,fp8,0,0.013438400626182557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,16,128,1,bfloat16,bfloat16,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,16,128,1,bfloat16,fp8,0,0.013140800595283508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,16,128,1,fp8,fp8,0,0.013174399733543396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,1,128,1,bfloat16,bfloat16,0,0.01218400001525879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,1,128,1,bfloat16,fp8,0,0.01284320056438446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,16,16,128,1,bfloat16,fp8,0,0.04971039891242981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,1,128,1,fp8,fp8,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,2,128,1,bfloat16,bfloat16,0,0.012241599708795547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,2,128,1,bfloat16,fp8,0,0.01276959925889969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,2,128,1,fp8,fp8,0,0.012828800082206725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,4,128,1,bfloat16,fp8,0,0.012833599746227265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,4,128,1,fp8,fp8,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,8,128,1,bfloat16,bfloat16,0,0.01223519966006279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,16,128,1,fp8,fp8,0,0.028654399514198303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,16,1,128,1,fp8,fp8,0,0.0146479994058609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,8,128,1,fp8,fp8,0,0.013006399571895599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,16,128,1,bfloat16,bfloat16,0,0.012265600264072418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,16,128,1,bfloat16,fp8,0,0.012886400520801543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,16,128,1,fp8,fp8,0,0.012851199507713318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,1,128,1,bfloat16,bfloat16,0,0.012283200025558471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,1,128,1,bfloat16,fp8,0,0.012583999335765839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,1,128,1,fp8,fp8,0,0.012947200238704682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,2,128,1,bfloat16,bfloat16,0,0.012064000219106674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,2,128,1,bfloat16,fp8,0,0.012943999469280243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,2,128,1,fp8,fp8,0,0.0126351997256279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,4,128,1,bfloat16,bfloat16,0,0.012345600128173827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,4,128,1,bfloat16,fp8,0,0.012654399871826172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,4,128,1,fp8,fp8,0,0.012958399951457977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,8,128,1,bfloat16,bfloat16,0,0.012006399780511856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,8,128,1,bfloat16,fp8,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,16,8,128,1,fp8,fp8,0,0.012585599720478059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,16,128,1,bfloat16,bfloat16,0,0.012070400267839431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,16,128,1,bfloat16,fp8,0,0.012558400630950928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,16,128,1,fp8,fp8,0,0.012531200051307678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,1,128,1,bfloat16,bfloat16,0,0.012014400213956833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,8,128,1,bfloat16,fp8,0,0.01374559998512268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,1,128,1,bfloat16,fp8,0,0.012732799351215362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,1,128,1,fp8,fp8,0,0.01271039992570877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,2,128,1,bfloat16,bfloat16,0,0.012198399752378464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,2,128,1,bfloat16,fp8,0,0.012734399735927581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,2,128,1,fp8,fp8,0,0.01266240030527115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,4,128,1,bfloat16,bfloat16,0,0.012123200297355651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,4,128,1,bfloat16,fp8,0,0.012748800218105316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,4,128,1,fp8,fp8,0,0.012539200484752655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,8,128,1,bfloat16,bfloat16,0,0.011935999989509583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,8,128,1,bfloat16,fp8,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,1,128,1,bfloat16,bfloat16,0,0.03336319923400879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,1,128,1,bfloat16,fp8,0,0.033846399188041686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,1,128,1,fp8,fp8,0,0.03450720012187958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,2,128,1,bfloat16,bfloat16,0,0.037427198886871335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,2,128,1,fp8,fp8,0,0.034113600850105286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,2,128,1,bfloat16,fp8,0,0.03499200046062469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,4,128,1,bfloat16,bfloat16,0,0.056492799520492555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,4,128,1,bfloat16,fp8,0,0.05384320020675659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,4,128,1,fp8,fp8,0,0.053958398103713986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,8,128,1,bfloat16,bfloat16,0,0.09158719778060913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,8,128,1,bfloat16,fp8,0,0.09512959718704224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,16,8,128,1,fp8,fp8,0,0.09689760208129883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,16,128,1,bfloat16,bfloat16,0,0.08615360260009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,16,128,1,bfloat16,fp8,0,0.08806560039520264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,16,128,1,fp8,fp8,0,0.08632799983024597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,1,128,1,bfloat16,fp8,0,0.02159679979085922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,1,128,1,fp8,fp8,0,0.021624000370502473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,2,128,1,bfloat16,bfloat16,0,0.020510399341583253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,2,128,1,bfloat16,fp8,0,0.021619200706481934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,4,128,1,bfloat16,bfloat16,0,0.012342400103807449
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,2,128,1,fp8,fp8,0,0.021635200083255767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,4,128,1,bfloat16,bfloat16,0,0.027263998985290527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,4,128,1,bfloat16,fp8,0,0.030563199520111085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,4,128,1,fp8,fp8,0,0.030086401104927062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,16,8,128,1,bfloat16,fp8,0,0.01308320015668869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,8,128,1,bfloat16,fp8,0,0.04812000095844269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,8,128,1,fp8,fp8,0,0.04806079864501953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,16,128,1,bfloat16,fp8,0,0.04706400036811829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,16,128,1,fp8,fp8,0,0.01844799965620041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,16,128,1,fp8,fp8,0,0.04705280065536499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,1,128,1,bfloat16,bfloat16,0,0.014406399428844452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,1,128,1,bfloat16,fp8,0,0.014913600683212281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,1,128,1,fp8,fp8,0,0.01510400027036667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,2,128,1,bfloat16,bfloat16,0,0.014316800236701965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,2,128,1,bfloat16,fp8,0,0.015275199711322785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,2,128,1,fp8,fp8,0,0.01494079977273941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,4,128,1,bfloat16,bfloat16,0,0.01825280040502548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,4,128,1,bfloat16,fp8,0,0.019259199500083923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,4,128,1,fp8,fp8,0,0.01958400011062622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,8,128,1,bfloat16,bfloat16,0,0.02579520046710968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,16,8,128,1,fp8,fp8,0,0.013572800159454345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,8,128,1,bfloat16,fp8,0,0.02852639853954315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,8,128,1,fp8,fp8,0,0.028288000822067262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,16,128,1,bfloat16,bfloat16,0,0.025484800338745117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,16,128,1,bfloat16,fp8,0,0.02766079902648926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,16,128,1,fp8,fp8,0,0.027832001447677612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,1,128,1,bfloat16,bfloat16,0,0.013395200669765472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,1,128,1,bfloat16,fp8,0,0.014151999354362487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,1,128,1,fp8,fp8,0,0.01422400027513504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,2,128,1,bfloat16,bfloat16,0,0.013483199477195739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,2,128,1,bfloat16,fp8,0,0.013955199718475341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,2,128,1,fp8,fp8,0,0.01446399986743927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,4,128,1,bfloat16,fp8,0,0.013780799508094788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,4,128,1,fp8,fp8,0,0.01406240016222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,8,128,1,bfloat16,bfloat16,0,0.01716800034046173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,8,128,1,bfloat16,fp8,0,0.018512000143527985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,8,128,1,fp8,fp8,0,0.01823360025882721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,16,128,1,bfloat16,bfloat16,0,0.01685439944267273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,16,128,1,bfloat16,fp8,0,0.017969599366188048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,16,128,1,fp8,fp8,0,0.017961600422859193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,1,128,1,bfloat16,bfloat16,0,0.01311360001564026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,1,128,1,bfloat16,fp8,0,0.013299199938774108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,1,128,1,fp8,fp8,0,0.013724799454212188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,2,128,1,bfloat16,bfloat16,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,2,128,1,bfloat16,fp8,0,0.01371839940547943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,2,128,1,fp8,fp8,0,0.013604800403118133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,4,128,1,bfloat16,bfloat16,0,0.012811200320720672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,4,128,1,bfloat16,fp8,0,0.013283200562000275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,4,128,1,fp8,fp8,0,0.013460800051689148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,8,128,1,bfloat16,bfloat16,0,0.01290079951286316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,8,128,1,bfloat16,fp8,0,0.013523200154304504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,16,8,128,1,fp8,fp8,0,0.01348000019788742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,16,128,1,bfloat16,bfloat16,0,0.012878400087356568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,16,128,1,bfloat16,fp8,0,0.013353599607944489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,1,128,1,bfloat16,bfloat16,0,0.019918400049209594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,16,128,1,fp8,fp8,0,0.013495999574661254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,1,128,1,bfloat16,bfloat16,0,0.012516799569129943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,1,128,1,bfloat16,fp8,0,0.013147200644016265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,1,128,1,fp8,fp8,0,0.013184000551700593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,2,128,1,bfloat16,bfloat16,0,0.01255040019750595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,2,128,1,bfloat16,fp8,0,0.01313920021057129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,2,128,1,fp8,fp8,0,0.012980799376964568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,4,128,1,bfloat16,fp8,0,0.013140800595283508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,4,128,1,fp8,fp8,0,0.012967999279499053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,8,128,1,bfloat16,bfloat16,0,0.012510399520397186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,8,128,1,bfloat16,fp8,0,0.013201600313186646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,8,128,1,fp8,fp8,0,0.013140800595283508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,16,128,1,bfloat16,bfloat16,0,0.012383999675512314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,16,128,1,bfloat16,fp8,0,0.013155199587345123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,16,128,1,fp8,fp8,0,0.01297760009765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,1,128,1,bfloat16,bfloat16,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,1,128,1,bfloat16,fp8,0,0.012859199941158295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,1,128,1,fp8,fp8,0,0.013036799430847169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,2,128,1,bfloat16,bfloat16,0,0.012249600142240524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,16,8,128,1,bfloat16,bfloat16,0,0.04434559941291809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,2,128,1,bfloat16,fp8,0,0.013048000633716583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,2,128,1,fp8,fp8,0,0.012736000120639801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,4,128,1,bfloat16,bfloat16,0,0.011856000125408172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,4,128,1,bfloat16,fp8,0,0.012777599692344665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,16,16,128,1,bfloat16,bfloat16,0,0.04189760088920593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,8,128,1,bfloat16,bfloat16,0,0.012100800126791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,8,128,1,bfloat16,fp8,0,0.012910400331020356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,8,128,1,fp8,fp8,0,0.012720000743865967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,16,128,1,bfloat16,bfloat16,0,0.012121599912643433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,16,128,1,bfloat16,fp8,0,0.012854400277137756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,16,128,1,fp8,fp8,0,0.012740799784660339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,1,128,1,bfloat16,bfloat16,0,0.01202080026268959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,1,128,1,bfloat16,fp8,0,0.012856000661849975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,1,128,1,fp8,fp8,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,2,128,1,bfloat16,bfloat16,0,0.012017600238323212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,2,128,1,bfloat16,fp8,0,0.01284479945898056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,2,128,1,fp8,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,4,128,1,bfloat16,bfloat16,0,0.011856000125408172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,4,128,1,bfloat16,fp8,0,0.01268640011548996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,4,128,1,fp8,fp8,0,0.012425599992275238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,8,128,1,bfloat16,bfloat16,0,0.011900799721479416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,8,128,1,bfloat16,fp8,0,0.012654399871826172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,16,8,128,1,fp8,fp8,0,0.01250240057706833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,16,128,1,bfloat16,bfloat16,0,0.012068799883127212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,16,128,1,fp8,fp8,0,0.012441600114107132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,1,128,1,bfloat16,bfloat16,0,0.011903999745845795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,1,128,1,bfloat16,fp8,0,0.01266079992055893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,1,128,1,fp8,fp8,0,0.012683199346065521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,2,128,1,bfloat16,bfloat16,0,0.011958400160074234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,2,128,1,bfloat16,fp8,0,0.012727999687194824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,2,128,1,fp8,fp8,0,0.01268479973077774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,4,128,1,bfloat16,bfloat16,0,0.011777599900960922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,4,128,1,bfloat16,fp8,0,0.0124719999730587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,4,128,1,fp8,fp8,0,0.01249760016798973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,8,128,1,bfloat16,bfloat16,0,0.01180799975991249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,16,8,128,1,fp8,fp8,0,0.012406399846076966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,8,128,1,bfloat16,fp8,0,0.012515200674533844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,8,128,1,fp8,fp8,0,0.012486399710178375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,16,4,128,1,bfloat16,bfloat16,0,0.013155199587345123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,1,128,1,bfloat16,bfloat16,0,0.02113119959831238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,1,128,1,bfloat16,fp8,0,0.022812800109386445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,1,128,1,fp8,fp8,0,0.023022399842739107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,2,128,1,bfloat16,bfloat16,0,0.028332799673080444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,2,128,1,bfloat16,fp8,0,0.03126719892024994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,2,128,1,fp8,fp8,0,0.03142240047454834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,4,128,1,bfloat16,bfloat16,0,0.043198400735855104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,4,128,1,bfloat16,fp8,0,0.04890719950199127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,4,128,1,fp8,fp8,0,0.048958399891853334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,8,128,1,bfloat16,bfloat16,0,0.07126240134239196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,8,128,1,bfloat16,fp8,0,0.0838479995727539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,16,8,128,1,fp8,fp8,0,0.08421279788017273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,16,128,1,bfloat16,bfloat16,0,0.06986079812049865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,16,128,1,bfloat16,fp8,0,0.08259199857711792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,1,128,1,bfloat16,bfloat16,0,0.015217599272727967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,16,128,1,fp8,fp8,0,0.08294559717178344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,1,128,1,bfloat16,fp8,0,0.01587039977312088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,1,128,1,fp8,fp8,0,0.016059200465679168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,2,128,1,bfloat16,bfloat16,0,0.01881600022315979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,2,128,1,bfloat16,fp8,0,0.02030719965696335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,2,128,1,fp8,fp8,0,0.020310400426387785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,4,128,1,bfloat16,bfloat16,0,0.0262719988822937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,4,128,1,bfloat16,fp8,0,0.02913439869880676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,4,128,1,fp8,fp8,0,0.02906079888343811
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,8,128,1,bfloat16,bfloat16,0,0.04070720076560974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,8,128,1,bfloat16,fp8,0,0.04657280147075653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,16,8,128,1,fp8,fp8,0,0.04663519859313965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,16,128,1,bfloat16,bfloat16,0,0.03953920006752014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,16,128,1,bfloat16,fp8,0,0.04568000137805939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,16,128,1,fp8,fp8,0,0.04563519954681396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,1,128,1,bfloat16,bfloat16,0,0.013366399705410004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,1,128,1,bfloat16,fp8,0,0.013952000439167023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,1,128,1,fp8,fp8,0,0.014079999923706055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,2,128,1,bfloat16,bfloat16,0,0.013321599364280701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,2,128,1,bfloat16,fp8,0,0.01406719982624054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,2,128,1,fp8,fp8,0,0.014169600605964661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,4,128,1,bfloat16,bfloat16,0,0.017289599776268004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,4,128,1,bfloat16,fp8,0,0.01855839937925339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,4,128,1,fp8,fp8,0,0.018769599497318268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,8,128,1,bfloat16,bfloat16,0,0.02462079972028732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,8,128,1,bfloat16,fp8,0,0.027340799570083618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,16,8,128,1,fp8,fp8,0,0.027529600262641906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,16,128,1,bfloat16,bfloat16,0,0.024278399348258973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,16,128,1,bfloat16,fp8,0,0.026830399036407472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,16,128,1,fp8,fp8,0,0.026972800493240356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,1,128,1,bfloat16,bfloat16,0,0.012971200048923492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,1,128,1,bfloat16,fp8,0,0.013459199666976928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,1,128,1,fp8,fp8,0,0.013521599769592284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,2,128,1,bfloat16,bfloat16,0,0.012673600018024445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,2,128,1,fp8,fp8,0,0.013467200100421906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,4,128,1,bfloat16,bfloat16,0,0.012718400359153748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,4,128,1,bfloat16,fp8,0,0.013633599877357483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,4,128,1,fp8,fp8,0,0.01364160031080246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,8,128,1,bfloat16,bfloat16,0,0.01640959978103638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,8,128,1,bfloat16,fp8,0,0.017876799404621124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,8,128,1,fp8,fp8,0,0.018055999279022218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,16,128,1,bfloat16,bfloat16,0,0.016100800037384032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,16,128,1,bfloat16,fp8,0,0.017407999932765962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,16,128,1,fp8,fp8,0,0.017532800137996674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,1,128,1,bfloat16,bfloat16,0,0.01189119964838028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,1,128,1,bfloat16,fp8,0,0.012768000364303589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,1,128,1,fp8,fp8,0,0.01281599998474121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,2,128,1,bfloat16,bfloat16,0,0.012092799693346024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,2,128,1,bfloat16,fp8,0,0.012966400384902954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,2,128,1,fp8,fp8,0,0.012806400656700134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,4,128,1,bfloat16,bfloat16,0,0.012140800058841706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,4,128,1,bfloat16,fp8,0,0.012996800243854523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,4,128,1,fp8,fp8,0,0.01281760036945343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,8,128,1,bfloat16,fp8,0,0.013315199315547943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,8,128,1,fp8,fp8,0,0.013023999333381654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,16,128,1,bfloat16,bfloat16,0,0.012700800597667695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,16,128,1,bfloat16,fp8,0,0.01305440068244934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,16,128,1,fp8,fp8,0,0.01316159963607788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,16,16,128,1,bfloat16,fp8,0,0.01266079992055893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,1,128,1,bfloat16,bfloat16,0,0.01175839975476265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,1,128,1,bfloat16,fp8,0,0.01247519999742508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,1,128,1,fp8,fp8,0,0.01263359934091568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,2,128,1,bfloat16,bfloat16,0,0.01196959987282753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,2,128,1,bfloat16,fp8,0,0.01273919939994812
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,2,128,1,fp8,fp8,0,0.01271200031042099
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,4,128,1,bfloat16,bfloat16,0,0.012201599776744843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,4,128,1,bfloat16,fp8,0,0.012895999848842621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,4,128,1,fp8,fp8,0,0.012929600477218629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,8,128,1,bfloat16,fp8,0,0.013059200346469879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,8,128,1,fp8,fp8,0,0.012969599664211273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,16,128,1,bfloat16,bfloat16,0,0.012220799922943115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,16,128,1,bfloat16,fp8,0,0.012931199371814727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,16,128,1,fp8,fp8,0,0.012905600666999816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,1,128,1,bfloat16,bfloat16,0,0.011659199744462967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,1,128,1,bfloat16,fp8,0,0.012404800206422806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,1,128,1,fp8,fp8,0,0.012678399682044983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,2,128,1,bfloat16,bfloat16,0,0.011737599968910217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,2,128,1,bfloat16,fp8,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,2,128,1,fp8,fp8,0,0.01255040019750595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,4,128,1,bfloat16,bfloat16,0,0.011662399768829346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,4,128,1,bfloat16,fp8,0,0.012563200294971466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,4,128,1,fp8,fp8,0,0.01250240057706833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,8,128,1,bfloat16,bfloat16,0,0.012171199917793274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,8,128,1,bfloat16,fp8,0,0.01279519945383072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,16,8,128,1,fp8,fp8,0,0.012782399356365205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,16,128,1,bfloat16,bfloat16,0,0.012150400131940842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,16,128,1,bfloat16,fp8,0,0.012727999687194824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,16,128,1,fp8,fp8,0,0.01276639997959137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,1,128,1,bfloat16,bfloat16,0,0.011670400202274323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,1,128,1,bfloat16,fp8,0,0.012492799758911132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,1,128,1,fp8,fp8,0,0.01244800016283989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,2,128,1,bfloat16,bfloat16,0,0.011724799871444702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,2,128,1,bfloat16,fp8,0,0.012433599680662155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,16,4,128,1,bfloat16,bfloat16,0,0.012300799787044524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,2,128,1,fp8,fp8,0,0.012334399670362473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,4,128,1,bfloat16,bfloat16,0,0.011681599915027619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,4,128,1,bfloat16,fp8,0,0.012406399846076966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,4,128,1,fp8,fp8,0,0.012372799962759019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,8,128,1,bfloat16,fp8,0,0.01239679977297783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,8,128,1,fp8,fp8,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,16,128,1,bfloat16,bfloat16,0,0.011767999827861786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,16,2,128,1,bfloat16,fp8,0,0.013348799943923951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,16,128,1,bfloat16,fp8,0,0.012318400293588638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,16,128,1,fp8,fp8,0,0.012375999987125397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,1,128,1,bfloat16,bfloat16,0,0.011519999802112579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,1,128,1,bfloat16,fp8,0,0.012123200297355651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,16,4,128,1,fp8,fp8,0,0.012627199292182922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,1,128,1,fp8,fp8,0,0.012272000312805176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,2,128,1,bfloat16,bfloat16,0,0.011374399811029435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,2,128,1,fp8,fp8,0,0.012278400361537933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,2,128,1,bfloat16,fp8,0,0.01223360002040863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,4,128,1,bfloat16,fp8,0,0.012486399710178375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,4,128,1,fp8,fp8,0,0.012243200093507767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,8,128,1,bfloat16,bfloat16,0,0.011715199798345566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,8,128,1,bfloat16,fp8,0,0.012270399928092956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,8,128,1,fp8,fp8,0,0.012432000041007996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,16,8,128,1,bfloat16,bfloat16,0,0.012204799801111221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,16,8,128,1,bfloat16,bfloat16,0,0.012404800206422806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,16,8,128,1,bfloat16,bfloat16,0,0.012222400307655335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,16,4,128,1,bfloat16,bfloat16,0,0.011401599645614624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,1,128,1,fp8,fp8,0,9.294537353515626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,1,128,1,bfloat16,fp8,0,9.479084777832032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,1,128,1,bfloat16,bfloat16,0,11.142704010009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,2,128,1,bfloat16,bfloat16,0,11.21506576538086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,2,128,1,bfloat16,fp8,0,9.350115203857422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,2,128,1,fp8,fp8,0,9.33522720336914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,4,128,1,bfloat16,fp8,0,9.394232177734375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,4,128,1,bfloat16,bfloat16,0,11.036329650878907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,12,128,1,bfloat16,bfloat16,0,6.096291351318359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,1,128,1,bfloat16,fp8,0,4.622024154663086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,1,128,1,bfloat16,bfloat16,0,5.703324890136718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,12,4,128,1,fp8,fp8,0,9.384398651123046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,1,128,1,fp8,fp8,0,4.62730712890625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,2,128,1,bfloat16,fp8,0,4.8710895538330075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,2,128,1,bfloat16,bfloat16,0,5.420286560058594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,2,128,1,fp8,fp8,0,4.652344131469727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,12,128,1,bfloat16,bfloat16,0,2.8228687286376952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,4,128,1,bfloat16,bfloat16,0,5.591387176513672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,4,128,1,bfloat16,fp8,0,4.689963150024414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,4,128,1,fp8,fp8,0,4.6815345764160154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,1,128,1,fp8,fp8,0,2.317030334472656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,1,128,1,bfloat16,fp8,0,2.4463903427124025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,1,128,1,bfloat16,bfloat16,0,2.906030464172363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,2,128,1,bfloat16,bfloat16,0,2.764364814758301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,2,128,1,bfloat16,fp8,0,2.333176040649414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,2,128,1,fp8,fp8,0,2.3418384552001954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,4,128,1,bfloat16,bfloat16,0,2.7926303863525392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,4,128,1,bfloat16,fp8,0,2.6185535430908202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,12,128,1,bfloat16,bfloat16,0,1.367636775970459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,4,128,1,fp8,fp8,0,2.4159536361694336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,1,128,1,bfloat16,bfloat16,0,1.3512895584106446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,1,128,1,bfloat16,fp8,0,1.3306879997253418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,1,128,1,fp8,fp8,0,1.340287971496582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,2,128,1,bfloat16,bfloat16,0,1.4076959609985351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,2,128,1,bfloat16,fp8,0,1.2788368225097657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,2,128,1,fp8,fp8,0,1.496395206451416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,4,128,1,bfloat16,bfloat16,0,1.3300175666809082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,4,128,1,bfloat16,fp8,0,1.3938223838806152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,4,128,1,fp8,fp8,0,1.619175910949707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,1,128,1,bfloat16,fp8,0,5.802833557128906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,1,128,1,bfloat16,bfloat16,0,6.8865501403808596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,1,128,1,fp8,fp8,0,5.355777740478516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,2,128,1,bfloat16,bfloat16,0,6.372663879394532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,2,128,1,bfloat16,fp8,0,5.414328002929688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,2,128,1,fp8,fp8,0,5.482988739013672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,4,128,1,bfloat16,bfloat16,0,7.015153503417968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,4,128,1,bfloat16,fp8,0,5.3942913055419925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,12,128,1,bfloat16,bfloat16,0,3.306932830810547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,12,4,128,1,fp8,fp8,0,5.498358535766601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,12,128,1,bfloat16,fp8,0,2.7941455841064453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,12,128,1,bfloat16,fp8,0,2.4310976028442384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,12,128,1,bfloat16,fp8,0,4.847008132934571
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,12,128,1,fp8,fp8,0,1.2345696449279786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,12,12,128,1,fp8,fp8,0,4.846625518798828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,12,128,1,fp8,fp8,0,2.792582321166992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,1,128,1,bfloat16,bfloat16,0,2.8526992797851562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,1,128,1,bfloat16,fp8,0,2.645305633544922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,1,128,1,fp8,fp8,0,2.6444639205932616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,2,128,1,bfloat16,bfloat16,0,2.9119327545166014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,12,12,128,1,bfloat16,fp8,0,1.2243200302124024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,2,128,1,bfloat16,fp8,0,2.6688880920410156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,2,128,1,fp8,fp8,0,2.6615840911865236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,12,128,1,bfloat16,fp8,0,1.406158447265625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,12,128,1,bfloat16,bfloat16,0,1.7356832504272461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,12,128,1,fp8,fp8,0,1.4165583610534669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,12,12,128,1,fp8,fp8,0,2.4352352142333986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,4,128,1,bfloat16,bfloat16,0,3.2744144439697265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,4,128,1,bfloat16,fp8,0,2.6592432022094727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,12,4,128,1,fp8,fp8,0,2.676678466796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,1,128,1,fp8,fp8,0,1.3263327598571777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,1,128,1,bfloat16,bfloat16,0,1.5011967658996581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,2,128,1,bfloat16,bfloat16,0,1.4571295738220216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,2,128,1,bfloat16,fp8,0,1.431764793395996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,2,128,1,fp8,fp8,0,1.3330991744995118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,4,128,1,bfloat16,bfloat16,0,1.5111968040466308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,12,128,1,bfloat16,bfloat16,0,0.8376175880432128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,12,128,1,bfloat16,fp8,0,0.7265151977539063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,12,128,1,fp8,fp8,0,0.7096367835998535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,4,128,1,bfloat16,fp8,0,1.3441264152526855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,1,128,1,bfloat16,bfloat16,0,0.7656479835510254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,4,128,1,fp8,fp8,0,1.580419158935547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,1,128,1,fp8,fp8,0,0.6683551788330078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,1,128,1,bfloat16,fp8,0,0.7090288162231445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,2,128,1,bfloat16,bfloat16,0,0.7227727890014648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,2,128,1,bfloat16,fp8,0,0.7144063949584961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,2,128,1,fp8,fp8,0,0.8284223556518555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,4,128,1,bfloat16,bfloat16,0,0.7590415954589844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,4,128,1,bfloat16,fp8,0,0.6779664039611817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,12,4,128,1,fp8,fp8,0,0.7052912235260009
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,12,1,128,1,bfloat16,fp8,0,1.444115161895752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,1,128,1,bfloat16,fp8,0,3.7495361328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,1,128,1,bfloat16,bfloat16,0,4.454780960083008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,1,128,1,fp8,fp8,0,3.954142379760742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,2,128,1,bfloat16,fp8,0,3.8102046966552736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,2,128,1,fp8,fp8,0,3.7716209411621096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,2,128,1,bfloat16,bfloat16,0,4.625860977172851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,4,128,1,bfloat16,bfloat16,0,4.370345687866211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,12,128,1,bfloat16,fp8,0,1.9740352630615234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,12,128,1,fp8,fp8,0,1.9779487609863282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,12,128,1,bfloat16,bfloat16,0,2.3332239151000977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,1,128,1,bfloat16,fp8,0,1.9812351226806642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,4,128,1,fp8,fp8,0,3.766328048706055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,1,128,1,fp8,fp8,0,2.0601408004760744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,2,128,1,bfloat16,fp8,0,1.8725551605224608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,2,128,1,bfloat16,bfloat16,0,2.1622047424316406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,2,128,1,fp8,fp8,0,2.0257919311523436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,4,128,1,bfloat16,bfloat16,0,2.2116575241088867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,4,128,1,bfloat16,fp8,0,1.8885263442993163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,12,128,1,bfloat16,bfloat16,0,1.1535200119018554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,12,128,1,bfloat16,fp8,0,1.1440272331237793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,12,128,1,fp8,fp8,0,1.0062352180480958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,12,4,128,1,bfloat16,fp8,0,3.770035171508789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,4,128,1,fp8,fp8,0,1.889031982421875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,12,1,128,1,bfloat16,bfloat16,0,2.017964744567871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,1,128,1,bfloat16,bfloat16,0,1.2260831832885741
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,1,128,1,bfloat16,fp8,0,0.9351967811584473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,1,128,1,fp8,fp8,0,1.069270420074463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,2,128,1,bfloat16,bfloat16,0,1.052779197692871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,2,128,1,bfloat16,fp8,0,0.9375103950500489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,2,128,1,fp8,fp8,0,0.9384592056274415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,4,128,1,bfloat16,bfloat16,0,1.0416303634643556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,12,128,1,bfloat16,bfloat16,0,0.599675178527832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,12,128,1,bfloat16,fp8,0,0.5587168216705323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,12,128,1,fp8,fp8,0,0.5578032016754151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,1,128,1,bfloat16,bfloat16,0,0.49937119483947756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,1,128,1,bfloat16,fp8,0,0.47246241569519043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,4,128,1,fp8,fp8,0,1.1087504386901856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,1,128,1,fp8,fp8,0,0.5117040157318116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,2,128,1,bfloat16,bfloat16,0,0.5130928039550782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,2,128,1,bfloat16,fp8,0,0.49509282112121583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,2,128,1,fp8,fp8,0,0.5079423904418945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,4,128,1,bfloat16,bfloat16,0,0.5142799854278565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,4,128,1,bfloat16,fp8,0,0.4809232234954834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,12,4,128,1,fp8,fp8,0,0.5282303810119628
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,12,4,128,1,bfloat16,fp8,0,0.9668736457824707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,1,128,1,bfloat16,fp8,0,5.025559997558593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,1,128,1,fp8,fp8,0,4.911119842529297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,2,128,1,bfloat16,fp8,0,5.013006210327148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,2,128,1,fp8,fp8,0,5.0400127410888675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,1,128,1,bfloat16,bfloat16,0,5.722016143798828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,2,128,1,bfloat16,bfloat16,0,5.943656158447266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,4,128,1,bfloat16,bfloat16,0,6.0577136993408205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,12,128,1,bfloat16,fp8,0,2.572427177429199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,12,128,1,bfloat16,bfloat16,0,2.9595455169677733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,12,128,1,fp8,fp8,0,2.5794704437255858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,1,128,1,bfloat16,bfloat16,0,2.826068878173828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,1,128,1,bfloat16,fp8,0,2.7510175704956055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,1,128,1,fp8,fp8,0,2.5658336639404298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,4,128,1,fp8,fp8,0,5.449086380004883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,2,128,1,fp8,fp8,0,2.4519392013549806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,2,128,1,bfloat16,fp8,0,2.829902458190918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,2,128,1,bfloat16,bfloat16,0,3.1644319534301757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,4,128,1,bfloat16,bfloat16,0,2.8733087539672852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,4,128,1,bfloat16,fp8,0,2.5863584518432616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,12,4,128,1,fp8,fp8,0,2.4781871795654298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,12,4,128,1,bfloat16,fp8,0,4.950163269042969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,12,128,1,bfloat16,fp8,0,1.3050559997558593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,12,128,1,bfloat16,bfloat16,0,1.4928223609924316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,12,128,1,fp8,fp8,0,1.3542384147644042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,1,128,1,bfloat16,bfloat16,0,1.6441551208496095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,1,128,1,bfloat16,fp8,0,1.3504768371582032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,1,128,1,fp8,fp8,0,1.2396080017089843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,2,128,1,bfloat16,fp8,0,1.2226528167724608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,2,128,1,fp8,fp8,0,1.232313632965088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,12,128,1,bfloat16,bfloat16,0,0.7348624229431152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,4,128,1,bfloat16,bfloat16,0,1.614780807495117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,4,128,1,bfloat16,fp8,0,1.2753952026367188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,12,128,1,fp8,fp8,0,0.7012479782104493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,4,128,1,fp8,fp8,0,1.4684703826904297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,1,128,1,bfloat16,bfloat16,0,0.6696000099182129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,1,128,1,fp8,fp8,0,0.6177552223205567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,1,128,1,bfloat16,fp8,0,0.8166383743286133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,2,128,1,bfloat16,bfloat16,0,0.676691198348999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,2,128,1,bfloat16,fp8,0,0.669916820526123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,2,128,1,fp8,fp8,0,0.6700064182281494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,4,128,1,bfloat16,bfloat16,0,0.6777679920196533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,4,128,1,bfloat16,fp8,0,0.8289631843566895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,4,128,1,fp8,fp8,0,0.6915247917175293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,12,128,1,bfloat16,bfloat16,0,0.3776479959487915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,12,128,1,bfloat16,fp8,0,0.35288639068603517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,12,128,1,fp8,fp8,0,0.3430016040802002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,1,128,1,bfloat16,bfloat16,0,0.4289584159851074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,1,128,1,fp8,fp8,0,0.3213567972183228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,1,128,1,bfloat16,fp8,0,0.3352672100067139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,2,128,1,bfloat16,fp8,0,0.32553279399871826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,2,128,1,fp8,fp8,0,0.3335567951202393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,4,128,1,bfloat16,bfloat16,0,0.3458559989929199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,4,128,1,fp8,fp8,0,0.32799839973449707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,12,2,128,1,bfloat16,bfloat16,0,1.4262175559997559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,12,12,128,1,bfloat16,fp8,0,0.6538688182830811
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,1,128,1,bfloat16,fp8,0,2.8530000686645507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,1,128,1,fp8,fp8,0,2.849510383605957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,1,128,1,bfloat16,bfloat16,0,3.2229598999023437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,2,128,1,bfloat16,bfloat16,0,0.3369999885559082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,2,128,1,bfloat16,bfloat16,0,3.3596416473388673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,12,4,128,1,bfloat16,fp8,0,0.3341952085494995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,2,128,1,bfloat16,fp8,0,2.825760078430176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,12,128,1,bfloat16,bfloat16,0,1.7479167938232423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,12,128,1,fp8,fp8,0,1.5399904251098633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,12,128,1,bfloat16,fp8,0,1.7318031311035156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,1,128,1,bfloat16,bfloat16,0,1.572532844543457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,2,128,1,fp8,fp8,0,2.879264068603516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,4,128,1,bfloat16,fp8,0,2.866561508178711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,4,128,1,fp8,fp8,0,3.0321935653686523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,12,4,128,1,bfloat16,bfloat16,0,3.5558032989501953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,1,128,1,bfloat16,fp8,0,1.5335552215576171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,1,128,1,fp8,fp8,0,1.4220352172851562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,2,128,1,bfloat16,bfloat16,0,1.588145637512207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,2,128,1,bfloat16,fp8,0,1.6050128936767578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,2,128,1,fp8,fp8,0,1.4475919723510742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,4,128,1,bfloat16,bfloat16,0,1.5832240104675293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,4,128,1,bfloat16,fp8,0,1.4563983917236327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,1,128,1,bfloat16,bfloat16,0,0.765657615661621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,12,128,1,fp8,fp8,0,0.8809408187866211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,1,128,1,bfloat16,fp8,0,0.7059279918670655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,2,128,1,bfloat16,bfloat16,0,0.7698991775512696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,1,128,1,fp8,fp8,0,0.8213328361511231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,2,128,1,bfloat16,fp8,0,0.7539487838745117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,2,128,1,fp8,fp8,0,0.7656415939331055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,4,128,1,bfloat16,bfloat16,0,0.7825376033782959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,12,128,1,bfloat16,bfloat16,0,0.46310720443725584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,12,128,1,bfloat16,fp8,0,0.41303839683532717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,4,128,1,bfloat16,fp8,0,0.7267168045043946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,12,128,1,fp8,fp8,0,0.3955343961715698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,4,128,1,fp8,fp8,0,0.7850607872009278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,1,128,1,bfloat16,bfloat16,0,0.3896559953689575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,1,128,1,bfloat16,fp8,0,0.37387518882751464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,1,128,1,fp8,fp8,0,0.3615008115768433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,2,128,1,bfloat16,bfloat16,0,0.38745439052581787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,2,128,1,bfloat16,fp8,0,0.3816512107849121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,2,128,1,fp8,fp8,0,0.37606239318847656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,4,128,1,bfloat16,fp8,0,0.36819360256195066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,12,128,1,bfloat16,bfloat16,0,0.24883840084075928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,4,128,1,bfloat16,bfloat16,0,0.4081120014190674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,12,128,1,bfloat16,fp8,0,0.2084320068359375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,12,128,1,fp8,fp8,0,0.20381600856781007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,1,128,1,bfloat16,bfloat16,0,0.20421919822692872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,1,128,1,bfloat16,fp8,0,0.19244320392608644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,1,128,1,fp8,fp8,0,0.19109920263290406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,2,128,1,bfloat16,bfloat16,0,0.2008687973022461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,2,128,1,bfloat16,fp8,0,0.1929967999458313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,4,128,1,bfloat16,fp8,0,0.19083839654922485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,4,128,1,bfloat16,bfloat16,0,0.20955359935760498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,12,128,1,bfloat16,fp8,0,0.7792448043823242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,12,12,128,1,bfloat16,bfloat16,0,0.8877535820007324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,12,4,128,1,fp8,fp8,0,1.6459199905395507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,1,128,1,bfloat16,fp8,0,2.683415985107422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,12,4,128,1,fp8,fp8,0,0.36820158958435056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,1,128,1,fp8,fp8,0,2.6851999282836916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,1,128,1,bfloat16,bfloat16,0,3.0054143905639648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,2,128,1,bfloat16,bfloat16,0,2.9401920318603514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,2,128,1,bfloat16,fp8,0,2.66278076171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,2,128,1,fp8,fp8,0,0.19317439794540406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,12,4,128,1,fp8,fp8,0,0.19589120149612427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,12,128,1,bfloat16,fp8,0,1.4777135848999023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,12,128,1,fp8,fp8,0,1.4911744117736816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,1,128,1,bfloat16,bfloat16,0,1.431220817565918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,2,128,1,fp8,fp8,0,2.672559928894043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,4,128,1,bfloat16,fp8,0,2.7353471755981444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,4,128,1,bfloat16,bfloat16,0,3.0916351318359374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,12,4,128,1,fp8,fp8,0,2.8754127502441404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,1,128,1,bfloat16,fp8,0,1.326028823852539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,2,128,1,fp8,fp8,0,1.339840030670166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,2,128,1,bfloat16,fp8,0,1.3385552406311034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,4,128,1,bfloat16,fp8,0,1.3816672325134278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,2,128,1,bfloat16,bfloat16,0,1.4593152046203612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,4,128,1,bfloat16,bfloat16,0,1.5563776016235351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,12,128,1,bfloat16,bfloat16,0,1.8005456924438477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,12,128,1,bfloat16,fp8,0,0.7451551914215088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,12,128,1,bfloat16,bfloat16,0,0.8470319747924805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,4,128,1,fp8,fp8,0,1.3590959548950194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,1,128,1,fp8,fp8,0,0.6693103790283204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,2,128,1,bfloat16,bfloat16,0,0.7211984157562256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,1,128,1,bfloat16,fp8,0,0.6701791763305665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,2,128,1,bfloat16,fp8,0,0.6766111850738525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,12,1,128,1,fp8,fp8,0,1.3275823593139648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,2,128,1,fp8,fp8,0,0.68962721824646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,12,128,1,fp8,fp8,0,0.7856224060058594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,4,128,1,bfloat16,fp8,0,0.6902272224426269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,4,128,1,bfloat16,bfloat16,0,0.7432144165039063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,12,128,1,bfloat16,bfloat16,0,0.430782413482666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,12,128,1,bfloat16,fp8,0,0.3872175931930542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,4,128,1,fp8,fp8,0,0.6854447841644287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,1,128,1,bfloat16,bfloat16,0,0.35761759281158445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,1,128,1,bfloat16,fp8,0,0.33942561149597167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,1,128,1,fp8,fp8,0,0.34180800914764403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,2,128,1,bfloat16,bfloat16,0,0.36975998878479005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,2,128,1,bfloat16,fp8,0,0.3420448064804077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,2,128,1,fp8,fp8,0,0.3450144052505493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,4,128,1,bfloat16,fp8,0,0.3506495952606201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,4,128,1,bfloat16,bfloat16,0,0.3765007972717285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,4,128,1,fp8,fp8,0,0.35767040252685545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,12,128,1,bfloat16,bfloat16,0,0.2193135976791382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,1,128,1,bfloat16,bfloat16,0,0.1834272027015686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,12,128,1,bfloat16,fp8,0,0.19694080352783203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,1,128,1,bfloat16,fp8,0,0.17909120321273803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,12,128,1,fp8,fp8,0,0.19928640127182007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,2,128,1,bfloat16,fp8,0,0.1771615982055664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,2,128,1,bfloat16,bfloat16,0,0.18583040237426757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,2,128,1,fp8,fp8,0,0.180731201171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,4,128,1,bfloat16,bfloat16,0,0.19874559640884398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,4,128,1,bfloat16,fp8,0,0.18091039657592772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,4,128,1,fp8,fp8,0,0.17876960039138795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,12,128,1,fp8,fp8,0,0.11899839639663697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,1,128,1,bfloat16,bfloat16,0,0.09902719855308532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,1,128,1,bfloat16,fp8,0,0.09528959989547729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,1,128,1,fp8,fp8,0,0.09533119797706605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,2,128,1,bfloat16,bfloat16,0,0.10019040107727051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,2,128,1,bfloat16,fp8,0,0.09553920030593872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,12,1,128,1,bfloat16,bfloat16,0,0.7497551918029786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,4,128,1,bfloat16,bfloat16,0,0.10520000457763672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,4,128,1,bfloat16,fp8,0,0.09637759923934937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,4,128,1,fp8,fp8,0,0.09777920246124268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,12,12,128,1,fp8,fp8,0,0.3886703968048096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,1,128,1,bfloat16,bfloat16,0,1.651425552368164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,1,128,1,bfloat16,fp8,0,1.5738351821899415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,1,128,1,fp8,fp8,0,1.5843071937561035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,12,1,128,1,fp8,fp8,0,0.17380479574203492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,2,128,1,bfloat16,bfloat16,0,1.6948959350585937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,12,128,1,bfloat16,fp8,0,0.12078560590744018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,12,128,1,bfloat16,bfloat16,0,0.13731839656829833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,2,128,1,bfloat16,fp8,0,1.6490320205688476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,2,128,1,fp8,fp8,0,1.589742374420166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,12,2,128,1,fp8,fp8,0,0.0956719994544983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,4,128,1,bfloat16,bfloat16,0,1.8042415618896483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,12,128,1,bfloat16,fp8,0,0.9194128036499023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,12,128,1,bfloat16,bfloat16,0,1.0551535606384277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,4,128,1,bfloat16,fp8,0,1.6595472335815429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,12,4,128,1,fp8,fp8,0,1.7224943161010742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,1,128,1,bfloat16,fp8,0,0.8010704040527343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,1,128,1,bfloat16,bfloat16,0,0.8291616439819336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,12,128,1,fp8,fp8,0,0.9381471633911133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,1,128,1,fp8,fp8,0,0.8969663619995117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,2,128,1,bfloat16,bfloat16,0,0.8530271530151368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,2,128,1,bfloat16,fp8,0,0.8100159645080567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,2,128,1,fp8,fp8,0,0.8163935661315918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,12,128,1,bfloat16,bfloat16,0,0.5320352077484131
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,4,128,1,bfloat16,bfloat16,0,0.9002767562866211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,4,128,1,bfloat16,fp8,0,0.8236063957214356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,12,128,1,bfloat16,fp8,0,0.5284383773803711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,12,4,128,1,fp8,fp8,0,0.8818880081176758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,1,128,1,bfloat16,bfloat16,0,0.4274735927581787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,12,128,1,fp8,fp8,0,0.5042431831359864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,1,128,1,bfloat16,fp8,0,0.41002402305603025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,1,128,1,fp8,fp8,0,0.4087423801422119
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,2,128,1,bfloat16,bfloat16,0,0.4463871955871582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,2,128,1,bfloat16,fp8,0,0.4081424236297607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,2,128,1,fp8,fp8,0,0.4304783821105957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,4,128,1,bfloat16,bfloat16,0,0.4579760074615479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,4,128,1,bfloat16,fp8,0,0.43080639839172363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,12,4,128,1,fp8,fp8,0,0.4378367900848389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,12,128,1,bfloat16,fp8,0,0.2531455993652344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,1,128,1,bfloat16,bfloat16,0,0.2170095920562744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,12,128,1,fp8,fp8,0,0.26704161167144774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,1,128,1,bfloat16,fp8,0,0.21613919734954834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,1,128,1,fp8,fp8,0,0.21332321166992188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,2,128,1,bfloat16,bfloat16,0,0.22835841178894042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,2,128,1,bfloat16,fp8,0,0.2203455924987793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,12,128,1,bfloat16,fp8,0,0.14166719913482667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,4,128,1,bfloat16,fp8,0,0.22427361011505126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,12,128,1,fp8,fp8,0,0.14136480093002318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,12,128,1,bfloat16,bfloat16,0,0.14727360010147095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,4,128,1,bfloat16,bfloat16,0,0.22988479137420653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,1,128,1,bfloat16,bfloat16,0,0.11905119419097901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,4,128,1,fp8,fp8,0,0.22347838878631593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,2,128,1,bfloat16,fp8,0,0.1124608039855957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,2,128,1,bfloat16,bfloat16,0,0.1190991997718811
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,2,128,1,fp8,fp8,0,0.11446720361709595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,4,128,1,bfloat16,fp8,0,0.11468640565872193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,4,128,1,bfloat16,bfloat16,0,0.12577919960021972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,12,128,1,bfloat16,fp8,0,0.0800320029258728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,12,128,1,fp8,fp8,0,0.08053600192070007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,1,128,1,bfloat16,bfloat16,0,0.06481279730796814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,1,128,1,bfloat16,fp8,0,0.06204800009727478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,1,128,1,fp8,fp8,0,0.062080001831054686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,2,128,1,bfloat16,bfloat16,0,0.06515039801597595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,2,128,1,bfloat16,fp8,0,0.06228960156440735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,12,128,1,bfloat16,bfloat16,0,0.27714240550994873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,2,128,1,fp8,fp8,0,0.06239839792251587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,4,128,1,bfloat16,bfloat16,0,0.06521599888801574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,4,128,1,bfloat16,fp8,0,0.06386560201644897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,4,128,1,fp8,fp8,0,0.06245120167732239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,12,2,128,1,fp8,fp8,0,0.2129823923110962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,1,128,1,bfloat16,fp8,0,0.11129440069198608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,1,128,1,fp8,fp8,0,0.11014080047607422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,1,128,1,bfloat16,fp8,0,1.5505423545837402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,1,128,1,fp8,fp8,0,1.5772047996520997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,12,4,128,1,fp8,fp8,0,0.11774400472640992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,12,12,128,1,bfloat16,bfloat16,0,0.09101120233535767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,2,128,1,bfloat16,fp8,0,1.572646427154541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,2,128,1,bfloat16,bfloat16,0,1.678763198852539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,2,128,1,fp8,fp8,0,1.5708784103393554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,4,128,1,bfloat16,bfloat16,0,1.8020368576049806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,4,128,1,bfloat16,fp8,0,1.6441616058349608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,1,128,1,fp8,fp8,0,0.7953983783721924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,1,128,1,bfloat16,bfloat16,0,0.8735343933105468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,12,128,1,bfloat16,bfloat16,0,1.0876223564147949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,12,128,1,fp8,fp8,0,0.9425984382629394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,12,128,1,bfloat16,fp8,0,0.983191967010498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,4,128,1,fp8,fp8,0,1.6429103851318358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,12,1,128,1,bfloat16,bfloat16,0,1.6737279891967773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,2,128,1,bfloat16,fp8,0,0.8154208183288574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,2,128,1,bfloat16,bfloat16,0,0.8375295639038086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,2,128,1,fp8,fp8,0,0.7969696044921875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,4,128,1,bfloat16,fp8,0,0.8389936447143554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,4,128,1,bfloat16,bfloat16,0,0.8858816146850585
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,4,128,1,fp8,fp8,0,0.8190320014953614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,12,128,1,bfloat16,bfloat16,0,0.5601568222045898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,1,128,1,bfloat16,bfloat16,0,0.4110191822052002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,1,128,1,bfloat16,fp8,0,0.4020048141479492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,2,128,1,bfloat16,fp8,0,0.4091792106628418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,12,128,1,fp8,fp8,0,0.49420480728149413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,12,128,1,bfloat16,fp8,0,0.4855616092681885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,12,128,1,bfloat16,bfloat16,0,0.2912512063980103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,4,128,1,bfloat16,fp8,0,0.42377758026123047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,12,128,1,bfloat16,fp8,0,0.2564239978790283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,4,128,1,fp8,fp8,0,0.4238560199737549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,4,128,1,bfloat16,bfloat16,0,0.4452847957611084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,12,1,128,1,bfloat16,fp8,0,0.7909167766571045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,12,128,1,fp8,fp8,0,0.25989279747009275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,1,128,1,bfloat16,bfloat16,0,0.21182239055633545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,1,128,1,bfloat16,fp8,0,0.20818560123443602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,1,128,1,fp8,fp8,0,0.20409760475158692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,2,128,1,bfloat16,bfloat16,0,0.21515200138092042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,2,128,1,bfloat16,fp8,0,0.21109600067138673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,2,128,1,fp8,fp8,0,0.20782880783081054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,4,128,1,bfloat16,bfloat16,0,0.23060960769653321
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,4,128,1,fp8,fp8,0,0.21367039680480956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,12,128,1,bfloat16,fp8,0,0.14125440120697022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,12,128,1,fp8,fp8,0,0.14220319986343383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,1,128,1,bfloat16,bfloat16,0,0.11052320003509522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,1,128,1,bfloat16,fp8,0,0.10912799835205078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,1,128,1,fp8,fp8,0,0.10967999696731567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,2,128,1,bfloat16,bfloat16,0,0.11310559511184692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,2,128,1,bfloat16,fp8,0,0.11024160385131836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,2,128,1,fp8,fp8,0,0.1111407995223999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,4,128,1,bfloat16,bfloat16,0,0.11795519590377808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,4,128,1,fp8,fp8,0,0.11512800455093383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,12,128,1,bfloat16,bfloat16,0,0.08602719902992248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,1,128,1,fp8,fp8,0,0.39806399345397947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,2,128,1,bfloat16,bfloat16,0,0.42118082046508787
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,12,128,1,fp8,fp8,0,0.07845119833946228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,1,128,1,bfloat16,fp8,0,0.058177602291107175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,1,128,1,fp8,fp8,0,0.05945119857788086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,2,128,1,bfloat16,bfloat16,0,0.06051200032234192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,2,128,1,fp8,fp8,0,0.05845119953155518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,12,2,128,1,fp8,fp8,0,0.4120319843292236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,4,128,1,bfloat16,bfloat16,0,0.06513760089874268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,4,128,1,bfloat16,fp8,0,0.05994079709053039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,4,128,1,fp8,fp8,0,0.05899199843406677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,12,128,1,bfloat16,bfloat16,0,0.04845759868621826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,12,128,1,bfloat16,fp8,0,0.04116959869861603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,12,128,1,fp8,fp8,0,0.041091200709342954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,1,128,1,bfloat16,fp8,0,0.03638559877872467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,1,128,1,fp8,fp8,0,0.03630400002002716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,12,128,1,bfloat16,bfloat16,0,0.16139520406723024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,2,128,1,bfloat16,bfloat16,0,0.0371535986661911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,2,128,1,bfloat16,fp8,0,0.036646398901939395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,2,128,1,fp8,fp8,0,0.03632160127162933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,4,128,1,bfloat16,bfloat16,0,0.038340801000595094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,4,128,1,bfloat16,fp8,0,0.036590400338172915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,4,128,1,fp8,fp8,0,0.03663839995861053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,12,4,128,1,bfloat16,fp8,0,0.1145359992980957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,12,128,1,bfloat16,fp8,0,0.07977920174598693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,1,128,1,bfloat16,bfloat16,0,0.9778047561645508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,1,128,1,bfloat16,bfloat16,0,0.06036480069160462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,12,2,128,1,bfloat16,fp8,0,0.059798401594161985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,1,128,1,bfloat16,fp8,0,0.9642175674438477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,1,128,1,fp8,fp8,0,0.981833553314209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,2,128,1,bfloat16,bfloat16,0,1.0327743530273437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,12,1,128,1,bfloat16,bfloat16,0,0.03687199950218201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,2,128,1,bfloat16,fp8,0,0.9956800460815429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,12,4,128,1,bfloat16,fp8,0,0.21427841186523439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,2,128,1,fp8,fp8,0,0.9937487602233886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,4,128,1,bfloat16,bfloat16,0,1.080630397796631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,4,128,1,bfloat16,fp8,0,1.0406047821044921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,12,128,1,bfloat16,bfloat16,0,0.6961103916168213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,12,4,128,1,fp8,fp8,0,1.04519681930542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,12,128,1,bfloat16,fp8,0,0.6114831924438476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,1,128,1,bfloat16,bfloat16,0,0.4946256160736084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,12,128,1,fp8,fp8,0,0.6076911926269531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,1,128,1,bfloat16,fp8,0,0.528713607788086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,1,128,1,fp8,fp8,0,0.49779520034790037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,2,128,1,fp8,fp8,0,0.5023136138916016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,2,128,1,bfloat16,fp8,0,0.5124080181121826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,2,128,1,bfloat16,bfloat16,0,0.5190639972686768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,4,128,1,bfloat16,fp8,0,0.5192480087280273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,4,128,1,bfloat16,bfloat16,0,0.544375991821289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,1,128,1,fp8,fp8,0,0.2506576061248779
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,1,128,1,bfloat16,fp8,0,0.25159358978271484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,12,128,1,bfloat16,fp8,0,0.30724000930786133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,12,128,1,bfloat16,bfloat16,0,0.3599056005477905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,12,4,128,1,fp8,fp8,0,0.5303664207458496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,2,128,1,bfloat16,bfloat16,0,0.26317760944366453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,2,128,1,bfloat16,fp8,0,0.25615200996398924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,2,128,1,fp8,fp8,0,0.25517280101776124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,4,128,1,bfloat16,bfloat16,0,0.28324639797210693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,4,128,1,bfloat16,fp8,0,0.26836960315704345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,12,128,1,bfloat16,bfloat16,0,0.18837120532989501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,12,128,1,bfloat16,fp8,0,0.1605280041694641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,12,128,1,fp8,fp8,0,0.15991040468215942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,4,128,1,fp8,fp8,0,0.2688175916671753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,1,128,1,bfloat16,bfloat16,0,0.135206401348114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,1,128,1,bfloat16,fp8,0,0.1325984001159668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,1,128,1,fp8,fp8,0,0.13186399936676024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,2,128,1,bfloat16,fp8,0,0.13390079736709595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,2,128,1,fp8,fp8,0,0.13647840023040772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,4,128,1,bfloat16,bfloat16,0,0.14842079877853392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,4,128,1,bfloat16,fp8,0,0.14017119407653808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,4,128,1,fp8,fp8,0,0.1411631941795349
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,12,128,1,bfloat16,bfloat16,0,0.10144480466842651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,12,128,1,bfloat16,fp8,0,0.08658400177955627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,12,128,1,fp8,fp8,0,0.08661119937896729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,1,128,1,bfloat16,bfloat16,0,0.07406079769134521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,1,128,1,bfloat16,fp8,0,0.07014560103416442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,1,128,1,fp8,fp8,0,0.06915839910507202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,2,128,1,bfloat16,bfloat16,0,0.07677919864654541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,2,128,1,bfloat16,fp8,0,0.07122880220413208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,2,128,1,fp8,fp8,0,0.07026079893112183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,4,128,1,bfloat16,bfloat16,0,0.08033279776573181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,12,128,1,bfloat16,fp8,0,0.04670239984989166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,12,128,1,bfloat16,bfloat16,0,0.06179839968681335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,4,128,1,fp8,fp8,0,0.07511360049247742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,12,128,1,fp8,fp8,0,0.04651199877262115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,1,128,1,bfloat16,bfloat16,0,0.0410975992679596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,1,128,1,fp8,fp8,0,0.040587198734283444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,2,128,1,bfloat16,bfloat16,0,0.04164640009403229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,2,128,1,bfloat16,fp8,0,0.04047360122203827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,2,128,1,fp8,fp8,0,0.04077920019626617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,4,128,1,bfloat16,bfloat16,0,0.04379999935626984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,4,128,1,bfloat16,fp8,0,0.04105919897556305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,12,128,1,bfloat16,bfloat16,0,0.033958399295806886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,12,128,1,bfloat16,fp8,0,0.031825599074363706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,12,128,1,fp8,fp8,0,0.031883201003074645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,1,128,1,bfloat16,bfloat16,0,0.030191999673843384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,1,128,1,bfloat16,fp8,0,0.030270400643348693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,1,128,1,fp8,fp8,0,0.030151998996734618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,2,128,1,bfloat16,bfloat16,0,0.030556800961494445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,2,128,1,bfloat16,fp8,0,0.03012320101261139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,2,128,1,fp8,fp8,0,0.03049919903278351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,4,128,1,bfloat16,bfloat16,0,0.03112959861755371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,4,128,1,bfloat16,fp8,0,0.03086079955101013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,12,4,128,1,fp8,fp8,0,0.030606400966644288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,1,128,1,bfloat16,bfloat16,0,0.25888800621032715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,12,12,128,1,fp8,fp8,0,0.3126015901565552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,12,2,128,1,bfloat16,bfloat16,0,0.13893280029296876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,1,128,1,bfloat16,bfloat16,0,1.1734527587890624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,1,128,1,bfloat16,fp8,0,1.1513888359069824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,12,4,128,1,bfloat16,fp8,0,0.07446560263633728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,1,128,1,fp8,fp8,0,1.1675328254699706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,2,128,1,bfloat16,bfloat16,0,1.213383960723877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,1,128,1,bfloat16,fp8,0,0.040345600247383116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,12,4,128,1,fp8,fp8,0,0.04115360081195831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,2,128,1,bfloat16,fp8,0,1.1730064392089843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,2,128,1,fp8,fp8,0,1.1877056121826173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,4,128,1,bfloat16,bfloat16,0,1.2977487564086914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,12,128,1,bfloat16,bfloat16,0,0.8526224136352539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,12,128,1,bfloat16,fp8,0,0.7467951774597168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,1,128,1,fp8,fp8,0,0.5927567958831788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,4,128,1,bfloat16,fp8,0,1.248465633392334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,1,128,1,bfloat16,bfloat16,0,0.5921504020690918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,1,128,1,bfloat16,fp8,0,0.5926015853881836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,12,128,1,fp8,fp8,0,0.7439328193664551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,12,4,128,1,fp8,fp8,0,1.2380800247192383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,2,128,1,bfloat16,bfloat16,0,0.6125072002410888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,2,128,1,bfloat16,fp8,0,0.599726390838623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,2,128,1,fp8,fp8,0,0.6027440071105957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,4,128,1,bfloat16,fp8,0,0.6330575942993164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,4,128,1,fp8,fp8,0,0.6314655780792237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,12,128,1,bfloat16,bfloat16,0,0.44034719467163086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,12,4,128,1,bfloat16,bfloat16,0,0.6557263851165771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,12,128,1,fp8,fp8,0,0.3780656099319458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,1,128,1,bfloat16,bfloat16,0,0.3083807945251465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,12,128,1,bfloat16,fp8,0,0.3777712106704712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,1,128,1,bfloat16,fp8,0,0.30344319343566895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,1,128,1,fp8,fp8,0,0.3021728038787842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,2,128,1,bfloat16,bfloat16,0,0.314134407043457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,2,128,1,bfloat16,fp8,0,0.3053567886352539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,2,128,1,fp8,fp8,0,0.3045903921127319
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,4,128,1,bfloat16,fp8,0,0.3209712028503418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,4,128,1,bfloat16,bfloat16,0,0.3391184091567993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,12,128,1,bfloat16,fp8,0,0.19833760261535643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,12,128,1,fp8,fp8,0,0.19710400104522705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,1,128,1,fp8,fp8,0,0.15911840200424193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,12,128,1,bfloat16,bfloat16,0,0.22838399410247803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,1,128,1,bfloat16,fp8,0,0.157151997089386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,1,128,1,bfloat16,bfloat16,0,0.15858720541000365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,2,128,1,bfloat16,fp8,0,0.16001440286636354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,4,128,1,bfloat16,bfloat16,0,0.1789088010787964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,4,128,1,bfloat16,fp8,0,0.16798239946365356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,4,128,1,fp8,fp8,0,0.16838879585266114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,12,128,1,bfloat16,bfloat16,0,0.12108160257339477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,1,128,1,bfloat16,bfloat16,0,0.08660320043563843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,12,128,1,fp8,fp8,0,0.10495840311050415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,1,128,1,bfloat16,fp8,0,0.08698559999465942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,1,128,1,fp8,fp8,0,0.08574720025062561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,2,128,1,bfloat16,bfloat16,0,0.08941760063171386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,2,128,1,bfloat16,fp8,0,0.0886784017086029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,4,128,1,bfloat16,bfloat16,0,0.09483519792556763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,2,128,1,fp8,fp8,0,0.08751360177993775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,4,128,1,bfloat16,fp8,0,0.09267839789390564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,4,128,1,fp8,fp8,0,0.09097279906272888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,12,128,1,bfloat16,bfloat16,0,0.0657039999961853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,12,128,1,bfloat16,fp8,0,0.05947039723396301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,12,128,1,fp8,fp8,0,0.05874720215797424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,1,128,1,bfloat16,bfloat16,0,0.046003198623657225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,1,128,1,bfloat16,fp8,0,0.047092801332473753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,1,128,1,fp8,fp8,0,0.04640640020370483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,2,128,1,bfloat16,fp8,0,0.04834080040454865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,2,128,1,fp8,fp8,0,0.0474368005990982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,12,4,128,1,fp8,fp8,0,0.3253727912902832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,4,128,1,bfloat16,fp8,0,0.048456001281738284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,4,128,1,fp8,fp8,0,0.049188798666000365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,12,128,1,bfloat16,bfloat16,0,0.03609119951725006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,12,128,1,bfloat16,fp8,0,0.03259040117263794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,12,128,1,fp8,fp8,0,0.0324319988489151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,1,128,1,bfloat16,bfloat16,0,0.02783679962158203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,1,128,1,bfloat16,fp8,0,0.028003200888633728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,2,128,1,fp8,fp8,0,0.16139520406723024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,1,128,1,fp8,fp8,0,0.028352001309394838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,2,128,1,bfloat16,bfloat16,0,0.02789599895477295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,2,128,1,bfloat16,fp8,0,0.030665600299835206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,4,128,1,bfloat16,bfloat16,0,0.02847039997577667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,12,12,128,1,bfloat16,fp8,0,0.1061903953552246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,4,128,1,bfloat16,fp8,0,0.031169599294662474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,4,128,1,fp8,fp8,0,0.030248001217842102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,12,128,1,bfloat16,fp8,0,0.03846080005168915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,12,128,1,bfloat16,bfloat16,0,0.03840799927711487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,12,128,1,fp8,fp8,0,0.037427198886871335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,1,128,1,bfloat16,fp8,0,0.032120001316070554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,1,128,1,fp8,fp8,0,0.03187519907951355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,2,128,1,bfloat16,bfloat16,0,0.032779198884963986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,1,128,1,bfloat16,bfloat16,0,0.030372801423072814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,2,128,1,bfloat16,fp8,0,0.034118399024009705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,2,128,1,fp8,fp8,0,0.03391520082950592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,4,128,1,fp8,fp8,0,0.03573279976844788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,4,128,1,bfloat16,fp8,0,0.03528000116348266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,2,128,1,bfloat16,bfloat16,0,0.04769600033760071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,1,128,1,bfloat16,bfloat16,0,0.731987190246582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,1,128,1,bfloat16,fp8,0,0.7466335773468018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,1,128,1,fp8,fp8,0,0.7522831916809082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,12,4,128,1,bfloat16,bfloat16,0,0.05434719920158386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,12,2,128,1,bfloat16,bfloat16,0,0.16696319580078126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,2,128,1,bfloat16,fp8,0,0.7607200145721436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,12,2,128,1,fp8,fp8,0,0.030151998996734618
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,2,128,1,fp8,fp8,0,0.7608543872833252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,2,128,1,bfloat16,bfloat16,0,0.7684944152832032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,4,128,1,bfloat16,fp8,0,0.8258192062377929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,4,128,1,bfloat16,bfloat16,0,0.8584223747253418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,1,128,1,bfloat16,bfloat16,0,0.36943199634552004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,12,4,128,1,fp8,fp8,0,0.8378031730651856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,1,128,1,fp8,fp8,0,0.3786832094192505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,12,128,1,fp8,fp8,0,0.5395567893981934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,2,128,1,bfloat16,bfloat16,0,0.39140160083770753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,2,128,1,fp8,fp8,0,0.38835361003875735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,4,128,1,bfloat16,bfloat16,0,0.4369919776916504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,4,128,1,bfloat16,fp8,0,0.4203807830810547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,12,128,1,bfloat16,bfloat16,0,0.3235680103302002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,4,128,1,fp8,fp8,0,0.41971840858459475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,12,128,1,bfloat16,fp8,0,0.2768336057662964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,1,128,1,bfloat16,bfloat16,0,0.19175039529800414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,12,128,1,fp8,fp8,0,0.27677600383758544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,1,128,1,bfloat16,fp8,0,0.19654560089111328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,1,128,1,fp8,fp8,0,0.19982080459594725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,2,128,1,bfloat16,bfloat16,0,0.20294559001922607
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,2,128,1,bfloat16,fp8,0,0.2049567937850952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,2,128,1,fp8,fp8,0,0.20162880420684814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,4,128,1,bfloat16,bfloat16,0,0.22827839851379395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,4,128,1,bfloat16,fp8,0,0.2177135944366455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,12,4,128,1,fp8,fp8,0,0.2204927921295166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,12,128,1,bfloat16,bfloat16,0,0.17072160243988038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,12,128,1,bfloat16,fp8,0,0.14737919569015503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,12,128,1,fp8,fp8,0,0.14653600454330445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,12,4,128,1,bfloat16,bfloat16,0,0.034561601281166074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,1,128,1,bfloat16,fp8,0,0.10565439462661744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,12,128,1,bfloat16,fp8,0,0.5426496028900146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,1,128,1,bfloat16,fp8,0,0.3795936107635498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,1,128,1,fp8,fp8,0,0.10621440410614014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,2,128,1,bfloat16,fp8,0,0.10848640203475952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,2,128,1,fp8,fp8,0,0.10811840295791626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,4,128,1,bfloat16,bfloat16,0,0.12061120271682739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,4,128,1,bfloat16,fp8,0,0.1166416049003601
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,4,128,1,fp8,fp8,0,0.11587040424346924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,12,128,1,bfloat16,bfloat16,0,0.09286720156669617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,2,128,1,bfloat16,fp8,0,0.3943072080612183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,12,128,1,bfloat16,fp8,0,0.07994719743728637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,12,128,1,fp8,fp8,0,0.07953280210494995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,1,128,1,fp8,fp8,0,0.059308797121047974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,1,128,1,bfloat16,fp8,0,0.0593392014503479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,2,128,1,bfloat16,bfloat16,0,0.06104159951210022
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,2,128,1,bfloat16,fp8,0,0.06104320287704468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,2,128,1,fp8,fp8,0,0.06102399826049805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,4,128,1,bfloat16,bfloat16,0,0.06711999773979187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,4,128,1,bfloat16,fp8,0,0.06453279852867126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,4,128,1,fp8,fp8,0,0.06513440012931823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,12,128,1,bfloat16,bfloat16,0,0.05292320251464844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,12,128,1,bfloat16,fp8,0,0.04582560062408447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,12,128,1,fp8,fp8,0,0.046000000834465024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,1,128,1,bfloat16,bfloat16,0,0.032076799869537355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,1,128,1,bfloat16,fp8,0,0.03457440137863159
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,1,128,1,fp8,fp8,0,0.0349839985370636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,2,128,1,bfloat16,bfloat16,0,0.033497598767280576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,2,128,1,bfloat16,fp8,0,0.035102400183677676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,2,128,1,fp8,fp8,0,0.03527199923992157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,4,128,1,bfloat16,bfloat16,0,0.03988640010356903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,4,128,1,bfloat16,fp8,0,0.035897600650787356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,12,4,128,1,fp8,fp8,0,0.03605920076370239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,12,128,1,bfloat16,bfloat16,0,0.030452799797058106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,12,128,1,bfloat16,fp8,0,0.02648319900035858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,12,128,1,fp8,fp8,0,0.026332798600196838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,1,128,1,bfloat16,bfloat16,0,0.022265599668025972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,1,128,1,bfloat16,fp8,0,0.024004800617694853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,1,128,1,fp8,fp8,0,0.02385759949684143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,2,128,1,bfloat16,bfloat16,0,0.022572800517082214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,2,128,1,bfloat16,fp8,0,0.024409599602222443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,2,128,1,fp8,fp8,0,0.02433439940214157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,4,128,1,bfloat16,bfloat16,0,0.02324959933757782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,4,128,1,bfloat16,fp8,0,0.024820800125598907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,12,4,128,1,fp8,fp8,0,0.02484479993581772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,12,128,1,bfloat16,bfloat16,0,0.019598400592803954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,12,128,1,bfloat16,fp8,0,0.020043200254440306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,12,128,1,fp8,fp8,0,0.020238399505615234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,1,128,1,bfloat16,bfloat16,0,0.01761920005083084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,1,128,1,bfloat16,fp8,0,0.018702399730682374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,1,128,1,fp8,fp8,0,0.019340799748897554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,2,128,1,bfloat16,bfloat16,0,0.017715199291706084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,2,128,1,bfloat16,fp8,0,0.018836799263954162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,2,128,1,fp8,fp8,0,0.01929599940776825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,4,128,1,bfloat16,bfloat16,0,0.018115200102329254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,4,128,1,bfloat16,fp8,0,0.018991999328136444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,12,4,128,1,fp8,fp8,0,0.01940480023622513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,12,128,1,bfloat16,bfloat16,0,0.01812800019979477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,12,128,1,bfloat16,fp8,0,0.019446399807929993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,12,128,1,fp8,fp8,0,0.01982239931821823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,1,128,1,bfloat16,bfloat16,0,0.017033599317073822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,1,128,1,bfloat16,fp8,0,0.018462400138378143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,1,128,1,fp8,fp8,0,0.01886879950761795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,2,128,1,bfloat16,bfloat16,0,0.017481599748134614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,2,128,1,bfloat16,fp8,0,0.01860959976911545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,2,128,1,fp8,fp8,0,0.018963199853897095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,4,128,1,bfloat16,bfloat16,0,0.017580799758434296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,4,128,1,bfloat16,fp8,0,0.01871519982814789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,12,4,128,1,fp8,fp8,0,0.019014400243759156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,1,128,1,bfloat16,bfloat16,0,0.30431039333343507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,1,128,1,bfloat16,bfloat16,0,0.10453120470046998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,1,128,1,bfloat16,fp8,0,0.3181024074554443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,1,128,1,fp8,fp8,0,0.3202672004699707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,12,12,128,1,bfloat16,bfloat16,0,0.6323423862457276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,12,2,128,1,bfloat16,bfloat16,0,0.10944479703903198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,2,128,1,bfloat16,bfloat16,0,0.3269216060638428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,2,128,1,bfloat16,fp8,0,0.3282000064849854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,4,128,1,bfloat16,bfloat16,0,0.37571039199829104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,4,128,1,bfloat16,fp8,0,0.3623231887817383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,4,128,1,fp8,fp8,0,0.36090080738067626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,12,128,1,bfloat16,bfloat16,0,0.3006576061248779
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,12,128,1,bfloat16,fp8,0,0.23730878829956054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,12,128,1,fp8,fp8,0,0.23726398944854737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,1,128,1,bfloat16,bfloat16,0,0.16081279516220093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,1,128,1,bfloat16,fp8,0,0.16614559888839722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,1,128,1,fp8,fp8,0,0.16615519523620606
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,2,128,1,bfloat16,fp8,0,0.17109279632568358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,2,128,1,fp8,fp8,0,0.17295360565185547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,4,128,1,bfloat16,fp8,0,0.1873088002204895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,4,128,1,fp8,fp8,0,0.18731839656829835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,12,128,1,bfloat16,bfloat16,0,0.15884319543838502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,12,128,1,bfloat16,fp8,0,0.1264415979385376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,1,128,1,bfloat16,bfloat16,0,0.08724640011787414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,1,128,1,bfloat16,fp8,0,0.08995519876480103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,1,128,1,fp8,fp8,0,0.08966400027275086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,2,128,1,bfloat16,bfloat16,0,0.0927232027053833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,2,128,1,bfloat16,fp8,0,0.09239360094070434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,2,128,1,fp8,fp8,0,0.0930400013923645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,4,128,1,bfloat16,bfloat16,0,0.10435359477996826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,4,128,1,bfloat16,fp8,0,0.10094879865646363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,4,128,1,fp8,fp8,0,0.09984959959983826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,12,128,1,bfloat16,bfloat16,0,0.08728640079498291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,12,128,1,bfloat16,fp8,0,0.0692736029624939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,12,128,1,fp8,fp8,0,0.06894879937171935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,1,128,1,bfloat16,bfloat16,0,0.05068320035934448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,1,128,1,bfloat16,fp8,0,0.050302398204803464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,1,128,1,fp8,fp8,0,0.05056639909744263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,2,128,1,bfloat16,bfloat16,0,0.05288640260696411
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,2,128,1,bfloat16,fp8,0,0.05240640044212341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,2,128,1,fp8,fp8,0,0.05227680206298828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,4,128,1,bfloat16,bfloat16,0,0.05908640027046204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,4,128,1,bfloat16,fp8,0,0.05604640245437622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,12,4,128,1,fp8,fp8,0,0.05601919889450073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,12,128,1,bfloat16,bfloat16,0,0.04827840030193329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,12,128,1,bfloat16,fp8,0,0.039934399724006656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,12,128,1,fp8,fp8,0,0.038889598846435544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,1,128,1,bfloat16,bfloat16,0,0.027665600180625916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,1,128,1,bfloat16,fp8,0,0.028233599662780762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,1,128,1,fp8,fp8,0,0.02808319926261902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,2,128,1,bfloat16,bfloat16,0,0.028233599662780762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,2,128,1,bfloat16,fp8,0,0.02821280062198639
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,12,2,128,1,fp8,fp8,0,0.32803680896759035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,2,128,1,fp8,fp8,0,0.028388801217079162
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,4,128,1,bfloat16,bfloat16,0,0.033923199772834776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,4,128,1,bfloat16,fp8,0,0.030107200145721436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,12,128,1,bfloat16,bfloat16,0,0.027667200565338133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,12,4,128,1,fp8,fp8,0,0.02990399897098541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,12,1,128,1,bfloat16,bfloat16,0,0.058987200260162354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,12,128,1,bfloat16,fp8,0,0.022518399357795715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,12,128,1,fp8,fp8,0,0.02289759963750839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,1,128,1,bfloat16,bfloat16,0,0.018771199882030486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,1,128,1,bfloat16,fp8,0,0.020956799387931824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,1,128,1,fp8,fp8,0,0.020571200549602507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,2,128,1,bfloat16,bfloat16,0,0.019780799746513367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,2,128,1,bfloat16,fp8,0,0.020611199736595153
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,2,128,1,fp8,fp8,0,0.021028800308704375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,4,128,1,bfloat16,fp8,0,0.021305599808692934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,4,128,1,fp8,fp8,0,0.021036800742149354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,12,128,1,bfloat16,bfloat16,0,0.016683200001716615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,12,128,1,bfloat16,fp8,0,0.016487999260425566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,12,128,1,fp8,fp8,0,0.016264000535011293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,1,128,1,bfloat16,bfloat16,0,0.014686399698257446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,1,128,1,bfloat16,fp8,0,0.01531040072441101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,1,128,1,fp8,fp8,0,0.01510239988565445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,2,128,1,bfloat16,bfloat16,0,0.014955200254917145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,2,128,1,bfloat16,fp8,0,0.01544319987297058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,2,128,1,fp8,fp8,0,0.015243199467658997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,4,128,1,bfloat16,fp8,0,0.015761600434780122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,4,128,1,fp8,fp8,0,0.01541920006275177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,12,128,1,bfloat16,bfloat16,0,0.015736000239849092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,12,128,1,bfloat16,fp8,0,0.015777599811553956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,12,128,1,fp8,fp8,0,0.016150400042533875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,12,12,128,1,fp8,fp8,0,0.12643200159072876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,1,128,1,bfloat16,bfloat16,0,0.014153599739074707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,1,128,1,bfloat16,fp8,0,0.015464000403881073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,1,128,1,fp8,fp8,0,0.015401600301265717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,2,128,1,bfloat16,bfloat16,0,0.014337599277496338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,2,128,1,bfloat16,fp8,0,0.015491199493408204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,2,128,1,fp8,fp8,0,0.015427200496196747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,4,128,1,bfloat16,bfloat16,0,0.01464959979057312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,4,128,1,bfloat16,fp8,0,0.015545600652694702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,12,4,128,1,fp8,fp8,0,0.01560479998588562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,12,128,1,bfloat16,bfloat16,0,0.01481119990348816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,12,128,1,bfloat16,fp8,0,0.015702399611473083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,12,128,1,fp8,fp8,0,0.01581919938325882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,1,128,1,bfloat16,bfloat16,0,0.01406400054693222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,1,128,1,bfloat16,fp8,0,0.015031999349594117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,1,128,1,fp8,fp8,0,0.015068799257278442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,2,128,1,bfloat16,bfloat16,0,0.014091199636459351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,2,128,1,fp8,fp8,0,0.015244799852371215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,4,128,1,bfloat16,bfloat16,0,0.014393599331378936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,4,128,1,bfloat16,fp8,0,0.01533920019865036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,4,128,1,fp8,fp8,0,0.015265600383281707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,1,128,1,bfloat16,bfloat16,0,0.15320160388946533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,1,128,1,bfloat16,fp8,0,0.16245919466018677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,1,128,1,fp8,fp8,0,0.16624319553375244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,2,128,1,bfloat16,bfloat16,0,0.1654288053512573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,2,128,1,bfloat16,fp8,0,0.17314239740371704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,2,128,1,fp8,fp8,0,0.1726304054260254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,4,128,1,bfloat16,bfloat16,0,0.19274239540100097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,12,4,128,1,bfloat16,bfloat16,0,0.020177599787712098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,4,128,1,bfloat16,fp8,0,0.19140640497207642
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,12,4,128,1,fp8,fp8,0,0.1926975965499878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,2,128,1,bfloat16,bfloat16,0,0.17066240310668945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,12,4,128,1,bfloat16,bfloat16,0,0.19555360078811646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,12,128,1,bfloat16,fp8,0,0.12916640043258668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,1,128,1,bfloat16,bfloat16,0,0.08505759835243225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,12,128,1,fp8,fp8,0,0.12934240102767944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,1,128,1,bfloat16,fp8,0,0.08882880210876465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,1,128,1,fp8,fp8,0,0.08905280232429505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,2,128,1,bfloat16,bfloat16,0,0.09030879735946655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,2,128,1,bfloat16,fp8,0,0.0917743980884552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,2,128,1,fp8,fp8,0,0.09157760143280029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,4,128,1,bfloat16,fp8,0,0.1004639983177185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,4,128,1,bfloat16,bfloat16,0,0.10437279939651489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,4,128,1,fp8,fp8,0,0.09991040229797363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,12,128,1,bfloat16,bfloat16,0,0.08838239908218384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,12,128,1,fp8,fp8,0,0.06898720264434814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,1,128,1,bfloat16,bfloat16,0,0.048876801133155824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,1,128,1,bfloat16,fp8,0,0.047660800814628604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,1,128,1,fp8,fp8,0,0.04811519980430603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,2,128,1,bfloat16,bfloat16,0,0.05146239995956421
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,2,128,1,bfloat16,fp8,0,0.049409601092338565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,12,2,128,1,bfloat16,fp8,0,0.015156799554824829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,2,128,1,fp8,fp8,0,0.0495824009180069
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,4,128,1,bfloat16,bfloat16,0,0.05721920132637024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,4,128,1,bfloat16,fp8,0,0.05291039943695068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,4,128,1,fp8,fp8,0,0.053320002555847165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,12,128,1,bfloat16,fp8,0,0.0355536013841629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,12,128,1,fp8,fp8,0,0.035596799850463864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,1,128,1,bfloat16,bfloat16,0,0.02431199997663498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,1,128,1,bfloat16,fp8,0,0.025455999374389648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,1,128,1,fp8,fp8,0,0.025516799092292784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,2,128,1,bfloat16,bfloat16,0,0.025574401021003723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,2,128,1,bfloat16,fp8,0,0.025643199682235718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,2,128,1,fp8,fp8,0,0.02560639977455139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,4,128,1,bfloat16,bfloat16,0,0.03250080049037933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,4,128,1,fp8,fp8,0,0.026158401370048524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,12,128,1,bfloat16,bfloat16,0,0.025339201092720032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,12,128,1,bfloat16,fp8,0,0.020380799472332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,12,128,1,fp8,fp8,0,0.02098879963159561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,12,12,128,1,bfloat16,bfloat16,0,0.16175520420074463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,1,128,1,bfloat16,bfloat16,0,0.01772159934043884
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,1,128,1,bfloat16,fp8,0,0.019172799587249757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,2,128,1,bfloat16,bfloat16,0,0.01804800033569336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,2,128,1,bfloat16,fp8,0,0.019332799315452575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,2,128,1,fp8,fp8,0,0.01886879950761795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,4,128,1,bfloat16,bfloat16,0,0.019211199879646302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,12,4,128,1,bfloat16,bfloat16,0,0.01528480052947998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,4,128,1,bfloat16,fp8,0,0.019313600659370423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,4,128,1,fp8,fp8,0,0.01947360038757324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,12,128,1,bfloat16,bfloat16,0,0.015335999429225922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,12,128,1,bfloat16,fp8,0,0.014502400159835815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,1,128,1,bfloat16,bfloat16,0,0.013148799538612366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,1,128,1,bfloat16,fp8,0,0.01383039951324463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,1,128,1,fp8,fp8,0,0.013779200613498688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,2,128,1,bfloat16,bfloat16,0,0.013443200290203095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,12,12,128,1,bfloat16,fp8,0,0.06848480105400086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,2,128,1,bfloat16,fp8,0,0.013870400190353394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,2,128,1,fp8,fp8,0,0.013736000657081604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,4,128,1,bfloat16,bfloat16,0,0.013515199720859527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,4,128,1,bfloat16,fp8,0,0.013876800239086152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,4,128,1,fp8,fp8,0,0.014060799777507783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,12,128,1,bfloat16,bfloat16,0,0.013763199746608733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,12,128,1,bfloat16,fp8,0,0.013972799479961395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,12,128,1,fp8,fp8,0,0.013801600039005279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,1,128,1,bfloat16,bfloat16,0,0.012727999687194824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,1,128,1,bfloat16,fp8,0,0.013475200533866883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,1,128,1,fp8,fp8,0,0.013348799943923951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,2,128,1,bfloat16,fp8,0,0.013500800728797913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,2,128,1,fp8,fp8,0,0.013300800323486328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,4,128,1,bfloat16,bfloat16,0,0.013568000495433807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,4,128,1,bfloat16,fp8,0,0.013451200723648072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,4,128,1,fp8,fp8,0,0.013804799318313599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,12,128,1,bfloat16,bfloat16,0,0.013281600177288055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,12,128,1,bfloat16,fp8,0,0.013620799779891968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,12,128,1,fp8,fp8,0,0.013843199610710144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,1,128,1,bfloat16,bfloat16,0,0.012399999797344208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,1,128,1,bfloat16,fp8,0,0.013407999277114868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,1,128,1,fp8,fp8,0,0.013368000090122224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,2,128,1,bfloat16,bfloat16,0,0.012495999783277511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,2,128,1,bfloat16,fp8,0,0.013353599607944489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,2,128,1,fp8,fp8,0,0.013376000523567199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,4,128,1,bfloat16,bfloat16,0,0.01260959953069687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,4,128,1,bfloat16,fp8,0,0.013348799943923951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,12,4,128,1,fp8,fp8,0,0.013395200669765472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,12,128,1,bfloat16,bfloat16,0,0.012601600587368011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,12,128,1,bfloat16,fp8,0,0.013571199774742127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,12,128,1,fp8,fp8,0,0.013329599797725678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,1,128,1,bfloat16,bfloat16,0,0.012111999839544297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,1,128,1,fp8,fp8,0,0.012807999551296235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,2,128,1,bfloat16,bfloat16,0,0.012083200365304947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,12,128,1,bfloat16,bfloat16,0,0.047030401229858396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,2,128,1,bfloat16,fp8,0,0.012939199805259705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,2,128,1,fp8,fp8,0,0.012756800651550293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,4,128,1,bfloat16,fp8,0,0.012753599882125854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,4,128,1,bfloat16,bfloat16,0,0.012347199767827988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,4,128,1,fp8,fp8,0,0.012716799974441528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,1,128,1,bfloat16,bfloat16,0,0.08552160263061523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,1,128,1,bfloat16,fp8,0,0.08988800048828124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,1,128,1,fp8,fp8,0,0.08955680131912232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,2,128,1,bfloat16,bfloat16,0,0.09311839938163757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,2,128,1,bfloat16,fp8,0,0.09266560077667237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,4,128,1,bfloat16,bfloat16,0,0.11573280096054077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,4,128,1,bfloat16,fp8,0,0.11745439767837525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,12,4,128,1,bfloat16,fp8,0,0.02611680030822754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,4,128,1,fp8,fp8,0,0.11782239675521851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,12,128,1,bfloat16,fp8,0,0.09652799963951111
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,12,128,1,fp8,fp8,0,0.09603999853134156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,12,1,128,1,fp8,fp8,0,0.018830400705337525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,1,128,1,bfloat16,bfloat16,0,0.04879199862480164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,1,128,1,bfloat16,fp8,0,0.04958080053329468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,1,128,1,fp8,fp8,0,0.04880479872226715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,2,128,1,bfloat16,bfloat16,0,0.052331197261810306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,12,12,128,1,fp8,fp8,0,0.014612799882888794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,2,128,1,fp8,fp8,0,0.05097759962081909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,2,128,1,bfloat16,fp8,0,0.05076000094413757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,4,128,1,bfloat16,bfloat16,0,0.06516479849815368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,4,128,1,bfloat16,fp8,0,0.06204800009727478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,4,128,1,fp8,fp8,0,0.06241919994354248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,12,128,1,bfloat16,bfloat16,0,0.058019202947616574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,12,128,1,bfloat16,fp8,0,0.0486847996711731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,12,128,1,fp8,fp8,0,0.049563199281692505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,1,128,1,bfloat16,bfloat16,0,0.024321599304676055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,1,128,1,bfloat16,fp8,0,0.026153600215911864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,1,128,1,fp8,fp8,0,0.026012799143791197
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,2,128,1,bfloat16,bfloat16,0,0.02584480047225952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,2,128,1,bfloat16,fp8,0,0.026212799549102783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,2,128,1,fp8,fp8,0,0.02588160037994385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,4,128,1,bfloat16,bfloat16,0,0.03622559905052185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,4,128,1,bfloat16,fp8,0,0.03070240020751953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,12,4,128,1,fp8,fp8,0,0.03038559854030609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,12,128,1,bfloat16,bfloat16,0,0.029348799586296083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,12,128,1,fp8,fp8,0,0.02498079985380173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,1,128,1,bfloat16,bfloat16,0,0.018223999440670012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,1,128,1,bfloat16,fp8,0,0.01934719979763031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,1,128,1,fp8,fp8,0,0.019364799559116363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,2,128,1,bfloat16,bfloat16,0,0.018376000225543976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,2,128,1,bfloat16,fp8,0,0.019339199364185333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,2,128,1,fp8,fp8,0,0.019355200231075287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,4,128,1,bfloat16,bfloat16,0,0.018743999302387238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,4,128,1,bfloat16,fp8,0,0.019543999433517457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,4,128,1,fp8,fp8,0,0.019529600441455842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,12,128,1,bfloat16,bfloat16,0,0.018111999332904815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,12,128,1,bfloat16,fp8,0,0.018831999599933626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,12,128,1,fp8,fp8,0,0.018571199476718904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,12,1,128,1,bfloat16,fp8,0,0.012943999469280243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,1,128,1,bfloat16,bfloat16,0,0.01358720064163208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,1,128,1,fp8,fp8,0,0.013913600146770478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,2,128,1,bfloat16,bfloat16,0,0.013827200233936309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,2,128,1,bfloat16,fp8,0,0.013903999328613281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,2,128,1,fp8,fp8,0,0.014134399592876434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,12,2,128,1,fp8,fp8,0,0.09219520092010498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,12,12,128,1,bfloat16,bfloat16,0,0.10391360521316528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,4,128,1,bfloat16,bfloat16,0,0.013540799915790557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,4,128,1,bfloat16,fp8,0,0.014159999787807465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,4,128,1,fp8,fp8,0,0.013926400244235993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,12,128,1,bfloat16,bfloat16,0,0.013462400436401368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,12,128,1,bfloat16,fp8,0,0.013947199285030364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,1,128,1,bfloat16,bfloat16,0,0.012611199915409089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,12,128,1,fp8,fp8,0,0.013980799913406372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,1,128,1,fp8,fp8,0,0.013643200695514678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,2,128,1,bfloat16,bfloat16,0,0.012771199643611907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,2,128,1,bfloat16,fp8,0,0.01377439945936203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,2,128,1,fp8,fp8,0,0.013862399756908417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,4,128,1,bfloat16,bfloat16,0,0.012775999307632447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,4,128,1,bfloat16,fp8,0,0.013704000413417817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,4,128,1,fp8,fp8,0,0.01372160017490387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,12,128,1,bfloat16,fp8,0,0.013580800592899322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,1,128,1,bfloat16,bfloat16,0,0.012488000094890594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,1,128,1,bfloat16,fp8,0,0.013180799782276154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,1,128,1,fp8,fp8,0,0.013177600502967835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,2,128,1,bfloat16,bfloat16,0,0.012652799487113953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,2,128,1,bfloat16,fp8,0,0.013264000415802002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,2,128,1,fp8,fp8,0,0.013356800377368926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,12,2,128,1,bfloat16,bfloat16,0,0.012966400384902954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,12,12,128,1,bfloat16,fp8,0,0.02492160052061081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,4,128,1,bfloat16,bfloat16,0,0.012627199292182922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,4,128,1,bfloat16,fp8,0,0.013120000064373017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,4,128,1,fp8,fp8,0,0.013227200508117676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,12,128,1,bfloat16,bfloat16,0,0.012590399384498597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,12,128,1,bfloat16,fp8,0,0.013335999846458436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,1,128,1,bfloat16,bfloat16,0,0.012240000069141388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,1,128,1,fp8,fp8,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,12,1,128,1,bfloat16,fp8,0,0.013998399674892425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,2,128,1,bfloat16,bfloat16,0,0.012177599966526032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,2,128,1,bfloat16,fp8,0,0.013097600638866424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,2,128,1,fp8,fp8,0,0.012753599882125854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,4,128,1,bfloat16,bfloat16,0,0.012408000230789185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,12,1,128,1,bfloat16,fp8,0,0.013574400544166565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,4,128,1,bfloat16,fp8,0,0.012919999659061432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,4,128,1,fp8,fp8,0,0.012870399653911591
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,12,128,1,bfloat16,bfloat16,0,0.012382400035858155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,12,128,1,bfloat16,fp8,0,0.01313759982585907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,12,128,1,fp8,fp8,0,0.01300159990787506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,1,128,1,bfloat16,bfloat16,0,0.01199679970741272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,1,128,1,bfloat16,fp8,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,1,128,1,fp8,fp8,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,12,128,1,bfloat16,bfloat16,0,0.01268479973077774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,2,128,1,bfloat16,bfloat16,0,0.012076800316572189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,2,128,1,bfloat16,fp8,0,0.012720000743865967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,2,128,1,fp8,fp8,0,0.012736000120639801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,12,12,128,1,fp8,fp8,0,0.013449600338935852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,4,128,1,bfloat16,fp8,0,0.012603199481964112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,1,128,1,bfloat16,bfloat16,0,0.05055360198020935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,1,128,1,bfloat16,fp8,0,0.04996480047702789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,1,128,1,fp8,fp8,0,0.05063040256500244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,2,128,1,bfloat16,bfloat16,0,0.06069599986076355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,12,128,1,fp8,fp8,0,0.013335999846458436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,2,128,1,bfloat16,fp8,0,0.06017439961433411
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,2,128,1,fp8,fp8,0,0.059995198249816896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,4,128,1,bfloat16,bfloat16,0,0.06610720157623291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,4,128,1,bfloat16,fp8,0,0.06406239867210388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,12,4,128,1,fp8,fp8,0,0.0636575996875763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,12,128,1,bfloat16,bfloat16,0,0.07902719974517822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,12,128,1,bfloat16,fp8,0,0.07812479734420777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,1,128,1,bfloat16,bfloat16,0,0.025313600897789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,1,128,1,bfloat16,fp8,0,0.0271232008934021
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,1,128,1,fp8,fp8,0,0.026756799221038817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,2,128,1,bfloat16,bfloat16,0,0.0291375994682312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,2,128,1,fp8,fp8,0,0.03108159899711609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,4,128,1,bfloat16,bfloat16,0,0.03621279895305633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,4,128,1,bfloat16,fp8,0,0.031228798627853393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,4,128,1,fp8,fp8,0,0.03181599974632263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,12,128,1,bfloat16,bfloat16,0,0.04027040004730224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,12,128,1,bfloat16,fp8,0,0.03921439945697784
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,12,128,1,fp8,fp8,0,0.039083200693130496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,1,128,1,bfloat16,bfloat16,0,0.018907199800014495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,1,128,1,bfloat16,fp8,0,0.01992959976196289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,1,128,1,fp8,fp8,0,0.019923199713230134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,2,128,1,bfloat16,bfloat16,0,0.018475200235843658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,2,128,1,bfloat16,fp8,0,0.020003199577331543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,2,128,1,fp8,fp8,0,0.019952000677585603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,4,128,1,bfloat16,bfloat16,0,0.01931679993867874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,4,128,1,bfloat16,fp8,0,0.020164799690246583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,12,4,128,1,fp8,fp8,0,0.020172800123691558
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,12,128,1,bfloat16,bfloat16,0,0.02227199971675873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,12,128,1,bfloat16,fp8,0,0.023635199666023253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,1,128,1,bfloat16,bfloat16,0,0.013540799915790557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,4,128,1,bfloat16,bfloat16,0,0.012145599722862244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,12,4,128,1,fp8,fp8,0,0.01279360055923462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,1,128,1,bfloat16,fp8,0,0.014176000654697419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,1,128,1,fp8,fp8,0,0.014496000111103058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,2,128,1,bfloat16,bfloat16,0,0.013515199720859527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,2,128,1,bfloat16,fp8,0,0.014321599900722504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,2,128,1,fp8,fp8,0,0.014404800534248353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,12,1,128,1,bfloat16,fp8,0,0.012972800433635712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,4,128,1,bfloat16,bfloat16,0,0.01371839940547943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,4,128,1,bfloat16,fp8,0,0.014232000708580017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,4,128,1,fp8,fp8,0,0.01443839967250824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,12,128,1,bfloat16,bfloat16,0,0.01706559956073761
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,12,128,1,bfloat16,fp8,0,0.01828639954328537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,12,128,1,fp8,fp8,0,0.07787520289421082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,12,128,1,fp8,fp8,0,0.018057599663734436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,1,128,1,bfloat16,bfloat16,0,0.013020800054073333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,1,128,1,bfloat16,fp8,0,0.01377439945936203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,2,128,1,bfloat16,bfloat16,0,0.012873600423336028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,1,128,1,fp8,fp8,0,0.013766400516033173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,4,128,1,bfloat16,bfloat16,0,0.012807999551296235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,2,128,1,bfloat16,fp8,0,0.013700799643993377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,4,128,1,bfloat16,fp8,0,0.013601599633693695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,12,128,1,bfloat16,bfloat16,0,0.012886400520801543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,12,12,128,1,fp8,fp8,0,0.023473599553108217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,12,128,1,bfloat16,fp8,0,0.013529600203037262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,1,128,1,bfloat16,bfloat16,0,0.012439999729394913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,1,128,1,fp8,fp8,0,0.013422399759292603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,2,128,1,bfloat16,bfloat16,0,0.012238399684429168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,2,128,1,fp8,fp8,0,0.013116799294948578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,4,128,1,bfloat16,bfloat16,0,0.012272000312805176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,4,128,1,bfloat16,fp8,0,0.01308320015668869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,4,128,1,fp8,fp8,0,0.013145600259304047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,12,128,1,bfloat16,fp8,0,0.013396799564361572
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,12,128,1,bfloat16,bfloat16,0,0.01249760016798973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,12,128,1,fp8,fp8,0,0.013262400031089782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,1,128,1,bfloat16,bfloat16,0,0.011998400092124939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,12,2,128,1,bfloat16,fp8,0,0.03131999969482422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,1,128,1,bfloat16,fp8,0,0.012759999930858612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,1,128,1,fp8,fp8,0,0.012654399871826172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,2,128,1,bfloat16,fp8,0,0.013006399571895599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,2,128,1,bfloat16,bfloat16,0,0.012267199903726577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,2,128,1,fp8,fp8,0,0.012958399951457977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,4,128,1,bfloat16,bfloat16,0,0.012065599858760833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,4,128,1,bfloat16,fp8,0,0.012619200348854064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,12,4,128,1,fp8,fp8,0,0.012559999525547028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,12,128,1,bfloat16,fp8,0,0.013185599446296692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,12,128,1,fp8,fp8,0,0.013147200644016265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,2,128,1,fp8,fp8,0,0.014116799831390381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,1,128,1,bfloat16,fp8,0,0.01265760064125061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,1,128,1,fp8,fp8,0,0.012647999823093415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,2,128,1,bfloat16,bfloat16,0,0.012009599804878235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,2,128,1,bfloat16,fp8,0,0.012732799351215362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,2,128,1,fp8,fp8,0,0.012556800246238708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,4,128,1,bfloat16,bfloat16,0,0.011990399658679962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,4,128,1,bfloat16,fp8,0,0.012355200201272964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,4,128,1,fp8,fp8,0,0.012548799812793731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,12,128,1,bfloat16,bfloat16,0,0.011972799897193909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,12,4,128,1,fp8,fp8,0,0.01372160017490387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,12,128,1,bfloat16,fp8,0,0.012982399761676788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,12,128,1,fp8,fp8,0,0.012865599989891053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,1,128,1,bfloat16,fp8,0,0.01281760036945343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,1,128,1,fp8,fp8,0,0.012561599910259246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,2,128,1,bfloat16,fp8,0,0.012588800489902496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,12,128,1,fp8,fp8,0,0.013801600039005279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,2,128,1,fp8,fp8,0,0.012611199915409089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,1,128,1,bfloat16,fp8,0,0.013203200697898865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,4,128,1,bfloat16,bfloat16,0,0.011801599711179733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,4,128,1,bfloat16,fp8,0,0.01247519999742508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,12,2,128,1,bfloat16,fp8,0,0.013326400518417358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,4,128,1,fp8,fp8,0,0.012483199685811996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,1,128,1,bfloat16,bfloat16,0,0.030406400561332703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,1,128,1,bfloat16,fp8,0,0.03340159952640533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,1,128,1,fp8,fp8,0,0.033055999875068666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,2,128,1,fp8,fp8,0,0.03304480016231537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,4,128,1,bfloat16,bfloat16,0,0.05281760096549988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,4,128,1,bfloat16,fp8,0,0.05082240104675293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,1,128,1,bfloat16,bfloat16,0,0.012057600170373916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,4,128,1,fp8,fp8,0,0.05081599950790405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,12,128,1,bfloat16,bfloat16,0,0.06363840103149414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,12,128,1,fp8,fp8,0,0.06711680293083191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,1,128,1,bfloat16,bfloat16,0,0.019444799423217772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,1,128,1,bfloat16,fp8,0,0.020857599377632142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,1,128,1,fp8,fp8,0,0.020883199572563172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,1,128,1,bfloat16,bfloat16,0,0.012054400146007537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,2,128,1,bfloat16,bfloat16,0,0.019497600197792054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,12,2,128,1,bfloat16,bfloat16,0,0.012118399888277055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,2,128,1,fp8,fp8,0,0.020875200629234314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,4,128,1,bfloat16,bfloat16,0,0.02738400101661682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,4,128,1,bfloat16,fp8,0,0.030004799365997314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,4,128,1,fp8,fp8,0,0.030188798904418945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,12,128,1,bfloat16,bfloat16,0,0.034174400568008426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,12,128,1,bfloat16,fp8,0,0.038140800595283506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,1,128,1,bfloat16,bfloat16,0,0.014284799993038177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,1,128,1,bfloat16,fp8,0,0.014824000000953675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,1,128,1,fp8,fp8,0,0.01505119949579239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,2,128,1,bfloat16,bfloat16,0,0.014494399726390838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,2,128,1,bfloat16,fp8,0,0.014902399480342865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,2,128,1,fp8,fp8,0,0.014977599680423736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,4,128,1,bfloat16,bfloat16,0,0.018279999494552612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,4,128,1,bfloat16,fp8,0,0.019569599628448488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,4,128,1,fp8,fp8,0,0.019601599872112276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,12,128,1,bfloat16,bfloat16,0,0.0215488001704216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,12,128,1,bfloat16,fp8,0,0.023521600663661955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,12,128,1,fp8,fp8,0,0.023371200263500213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,1,128,1,bfloat16,bfloat16,0,0.013358399271965027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,2,128,1,bfloat16,bfloat16,0,0.03136000037193298
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,1,128,1,bfloat16,fp8,0,0.014107200503349304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,1,128,1,fp8,fp8,0,0.014009599387645722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,2,128,1,bfloat16,bfloat16,0,0.013449600338935852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,2,128,1,bfloat16,fp8,0,0.013825599849224091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,2,128,1,fp8,fp8,0,0.01395999938249588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,4,128,1,bfloat16,bfloat16,0,0.013199999928474426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,4,128,1,bfloat16,fp8,0,0.013976000249385834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,12,4,128,1,fp8,fp8,0,0.014008000493049622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,12,128,1,bfloat16,bfloat16,0,0.01669439971446991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,12,128,1,bfloat16,fp8,0,0.01793919950723648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,12,12,128,1,bfloat16,bfloat16,0,0.012361600250005721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,12,128,1,fp8,fp8,0,0.017931200563907623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,1,128,1,bfloat16,bfloat16,0,0.012484800070524216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,1,128,1,bfloat16,fp8,0,0.013254399597644805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,1,128,1,fp8,fp8,0,0.01334560066461563
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,2,128,1,bfloat16,bfloat16,0,0.012417600303888322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,2,128,1,bfloat16,fp8,0,0.01297760009765625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,2,128,1,fp8,fp8,0,0.013121600449085235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,4,128,1,bfloat16,bfloat16,0,0.012513600289821625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,4,128,1,bfloat16,fp8,0,0.01311199963092804
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,12,4,128,1,fp8,fp8,0,0.01311360001564026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,12,128,1,bfloat16,bfloat16,0,0.012904000282287598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,12,128,1,bfloat16,fp8,0,0.01361600011587143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,12,128,1,fp8,fp8,0,0.013628800213336945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,1,128,1,bfloat16,fp8,0,0.013116799294948578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,1,128,1,fp8,fp8,0,0.013184000551700593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,2,128,1,bfloat16,bfloat16,0,0.012179200351238251
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,2,128,1,bfloat16,fp8,0,0.012969599664211273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,2,128,1,fp8,fp8,0,0.01295199990272522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,4,128,1,bfloat16,bfloat16,0,0.012414400279521943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,4,128,1,bfloat16,fp8,0,0.013030399382114411
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,4,128,1,fp8,fp8,0,0.013055999577045441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,12,128,1,bfloat16,bfloat16,0,0.012403199821710587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,2,128,1,bfloat16,fp8,0,0.021191999316215515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,12,128,1,bfloat16,fp8,0,0.01337919980287552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,12,128,1,fp8,fp8,0,0.013180799782276154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,1,128,1,fp8,fp8,0,0.012587200105190276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,1,128,1,bfloat16,bfloat16,0,0.01199679970741272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,2,128,1,bfloat16,bfloat16,0,0.011868800222873687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,2,128,1,bfloat16,fp8,0,0.01257600039243698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,2,128,1,fp8,fp8,0,0.012449599802494049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,4,128,1,bfloat16,bfloat16,0,0.012057600170373916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,4,128,1,bfloat16,fp8,0,0.012515200674533844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,4,128,1,fp8,fp8,0,0.012697599828243256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,12,128,1,bfloat16,bfloat16,0,0.01234399974346161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,12,128,1,bfloat16,fp8,0,0.013371199369430542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,12,128,1,fp8,fp8,0,0.013180799782276154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,1,128,1,bfloat16,bfloat16,0,0.012062399834394454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,1,128,1,bfloat16,fp8,0,0.012614400684833526
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,1,128,1,fp8,fp8,0,0.012755200266838074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,2,128,1,bfloat16,bfloat16,0,0.011801599711179733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,2,128,1,bfloat16,fp8,0,0.012665599584579468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,4,128,1,bfloat16,bfloat16,0,0.01178399994969368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,2,128,1,fp8,fp8,0,0.012537600100040435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,4,128,1,bfloat16,fp8,0,0.012630400061607362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,12,4,128,1,fp8,fp8,0,0.012399999797344208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,12,128,1,bfloat16,bfloat16,0,0.01231520026922226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,12,128,1,bfloat16,fp8,0,0.012958399951457977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,12,128,1,fp8,fp8,0,0.012703999876976013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,1,128,1,bfloat16,bfloat16,0,0.011987199634313583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,1,128,1,bfloat16,fp8,0,0.012649600207805634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,1,128,1,fp8,fp8,0,0.012377600371837615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,2,128,1,bfloat16,bfloat16,0,0.011852800101041793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,2,128,1,bfloat16,fp8,0,0.012518399953842163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,2,128,1,fp8,fp8,0,0.012270399928092956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,4,128,1,bfloat16,bfloat16,0,0.011713600158691407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,4,128,1,bfloat16,fp8,0,0.012203200161457062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,12,4,128,1,fp8,fp8,0,0.012385600060224534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,12,1,128,1,bfloat16,bfloat16,0,0.020744000375270844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,12,1,128,1,bfloat16,fp8,0,0.02297919988632202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,12,1,128,1,fp8,fp8,0,0.022276799380779266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,12,2,128,1,bfloat16,bfloat16,0,0.028483200073242187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,12,2,128,1,bfloat16,fp8,0,0.03102880120277405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,12,2,128,1,fp8,fp8,0,0.031560000777244565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,12,4,128,1,bfloat16,bfloat16,0,0.042715200781822206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,12,4,128,1,bfloat16,fp8,0,0.049481600522994995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,12,2,128,1,bfloat16,fp8,0,0.033344000577926636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,12,4,128,1,fp8,fp8,0,0.048721599578857425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,12,128,1,bfloat16,bfloat16,0,0.05575360059738159
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,12,128,1,bfloat16,fp8,0,0.06474720239639283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,12,128,1,fp8,fp8,0,0.06494399905204773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,1,128,1,bfloat16,fp8,0,0.015806399285793304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,1,128,1,fp8,fp8,0,0.015727999806404113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,2,128,1,bfloat16,bfloat16,0,0.01847680062055588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,2,128,1,bfloat16,fp8,0,0.02008160054683685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,2,128,1,fp8,fp8,0,0.0199535995721817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,4,128,1,bfloat16,bfloat16,0,0.02594879865646362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,4,128,1,bfloat16,fp8,0,0.028838399052619933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,4,128,1,fp8,fp8,0,0.02895680069923401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,12,128,1,bfloat16,bfloat16,0,0.03215200006961823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,12,128,1,bfloat16,fp8,0,0.03686240017414093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,12,128,1,fp8,fp8,0,0.03686400055885315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,12,12,128,1,bfloat16,fp8,0,0.06685600280761719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,1,128,1,bfloat16,fp8,0,0.014112000167369843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,1,128,1,fp8,fp8,0,0.013913600146770478
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,2,128,1,bfloat16,bfloat16,0,0.013523200154304504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,12,1,128,1,bfloat16,bfloat16,0,0.012622399628162384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,2,128,1,bfloat16,fp8,0,0.014268800616264343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,2,128,1,fp8,fp8,0,0.014315199851989747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,4,128,1,bfloat16,bfloat16,0,0.017475199699401856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,4,128,1,bfloat16,fp8,0,0.01874080002307892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,4,128,1,fp8,fp8,0,0.01886879950761795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,12,128,1,bfloat16,bfloat16,0,0.020689600706100465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,12,128,1,bfloat16,fp8,0,0.02274399995803833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,12,128,1,fp8,fp8,0,0.022588799893856048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,1,128,1,bfloat16,bfloat16,0,0.012612800300121307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,1,128,1,bfloat16,fp8,0,0.013425600528717042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,1,128,1,fp8,fp8,0,0.013128000497817992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,2,128,1,bfloat16,bfloat16,0,0.012537600100040435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,2,128,1,bfloat16,fp8,0,0.013447999954223633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,2,128,1,fp8,fp8,0,0.013264000415802002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,4,128,1,bfloat16,bfloat16,0,0.012667199969291687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,4,128,1,bfloat16,fp8,0,0.013438400626182557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,12,4,128,1,fp8,fp8,0,0.013387200236320496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,12,128,1,bfloat16,bfloat16,0,0.016233600676059723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,12,1,128,1,bfloat16,fp8,0,0.01276959925889969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,12,128,1,bfloat16,fp8,0,0.01754239946603775
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,12,128,1,fp8,fp8,0,0.01746560037136078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,1,128,1,bfloat16,bfloat16,0,0.012164799869060517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,1,128,1,bfloat16,fp8,0,0.012899200618267059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,1,128,1,fp8,fp8,0,0.012695999443531036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,2,128,1,bfloat16,fp8,0,0.013182400166988373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,2,128,1,fp8,fp8,0,0.013062399625778199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,4,128,1,bfloat16,bfloat16,0,0.012217599898576736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,4,128,1,bfloat16,fp8,0,0.013222399353981017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,12,12,128,1,fp8,fp8,0,0.03784160017967224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,4,128,1,fp8,fp8,0,0.013075199723243714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,12,128,1,bfloat16,bfloat16,0,0.012355200201272964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,12,128,1,bfloat16,fp8,0,0.013457599282264709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,1,128,1,bfloat16,bfloat16,0,0.011844799667596818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,1,128,1,bfloat16,fp8,0,0.012495999783277511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,1,128,1,fp8,fp8,0,0.012548799812793731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,2,128,1,bfloat16,bfloat16,0,0.011806400120258331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,2,128,1,bfloat16,fp8,0,0.012428800016641617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,2,128,1,fp8,fp8,0,0.012676799297332763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,4,128,1,bfloat16,bfloat16,0,0.012009599804878235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,4,128,1,bfloat16,fp8,0,0.01268640011548996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,4,128,1,fp8,fp8,0,0.012868799269199371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,12,128,1,bfloat16,bfloat16,0,0.012350399792194367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,12,128,1,bfloat16,fp8,0,0.013105599582195282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,12,128,1,fp8,fp8,0,0.01313920021057129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,1,128,1,bfloat16,bfloat16,0,0.0115167997777462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,1,128,1,bfloat16,fp8,0,0.01231359988451004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,1,128,1,fp8,fp8,0,0.012334399670362473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,2,128,1,bfloat16,bfloat16,0,0.011604800075292587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,2,128,1,bfloat16,fp8,0,0.0123648002743721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,2,128,1,fp8,fp8,0,0.012268800288438797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,4,128,1,bfloat16,bfloat16,0,0.01183359995484352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,4,128,1,bfloat16,fp8,0,0.012430399656295776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,12,4,128,1,fp8,fp8,0,0.01242239996790886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,12,128,1,bfloat16,bfloat16,0,0.012468799948692322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,12,128,1,bfloat16,fp8,0,0.013007999956607818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,12,128,1,fp8,fp8,0,0.012891200184822083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,1,128,1,bfloat16,bfloat16,0,0.011526399850845337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,1,128,1,bfloat16,fp8,0,0.012054400146007537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,1,128,1,fp8,fp8,0,0.012252800166606903
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,2,128,1,bfloat16,bfloat16,0,0.011590400338172912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,2,128,1,bfloat16,fp8,0,0.01218079999089241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,2,128,1,fp8,fp8,0,0.012401600182056428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,4,128,1,bfloat16,bfloat16,0,0.011671999841928482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,4,128,1,bfloat16,fp8,0,0.01218400001525879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,12,4,128,1,fp8,fp8,0,0.012479999661445617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,12,128,1,bfloat16,bfloat16,0,0.012011200189590454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,12,128,1,bfloat16,fp8,0,0.01266240030527115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,12,128,1,fp8,fp8,0,0.012756800651550293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,1,128,1,bfloat16,bfloat16,0,0.011398400366306304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,1,128,1,bfloat16,fp8,0,0.012316799908876418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,1,128,1,fp8,fp8,0,0.012191999703645706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,2,128,1,bfloat16,bfloat16,0,0.011614400148391723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,2,128,1,bfloat16,fp8,0,0.012195199728012085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,2,128,1,fp8,fp8,0,0.012263999879360199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,4,128,1,bfloat16,bfloat16,0,0.011472000181674958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,4,128,1,bfloat16,fp8,0,0.01228799968957901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,12,1,128,1,bfloat16,bfloat16,0,0.014710399508476257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,12,4,128,1,fp8,fp8,0,0.012172800302505494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,12,1,128,1,bfloat16,bfloat16,0,0.013169600069522858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,12,2,128,1,bfloat16,bfloat16,0,0.01186240017414093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,12,12,128,1,fp8,fp8,0,0.013492800295352936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,1,128,1,fp8,fp8,0,6.198468780517578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,1,128,1,bfloat16,fp8,0,6.22217903137207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,2,128,1,bfloat16,fp8,0,6.238684844970703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,1,128,1,bfloat16,bfloat16,0,7.2891395568847654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,2,128,1,bfloat16,bfloat16,0,7.364497375488281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,2,128,1,fp8,fp8,0,6.2092735290527346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,4,128,1,bfloat16,bfloat16,0,7.653031921386718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,8,128,1,bfloat16,bfloat16,0,3.824039840698242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,8,128,1,bfloat16,fp8,0,3.2183502197265623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,8,128,1,fp8,fp8,0,3.3276016235351564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,1,128,1,bfloat16,fp8,0,3.0944591522216798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,1,128,1,bfloat16,bfloat16,0,3.662615966796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,4,128,1,bfloat16,fp8,0,6.377187347412109
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,8,4,128,1,fp8,fp8,0,6.2867168426513675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,1,128,1,fp8,fp8,0,3.2887054443359376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,2,128,1,bfloat16,fp8,0,3.108569526672363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,2,128,1,fp8,fp8,0,3.1075424194335937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,2,128,1,bfloat16,bfloat16,0,3.687424087524414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,8,128,1,bfloat16,bfloat16,0,1.8115999221801757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,4,128,1,bfloat16,fp8,0,3.2849281311035154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,8,128,1,bfloat16,fp8,0,1.6211967468261719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,4,128,1,bfloat16,bfloat16,0,3.7391342163085937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,8,4,128,1,fp8,fp8,0,3.156065559387207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,8,128,1,fp8,fp8,0,1.6195871353149414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,1,128,1,bfloat16,bfloat16,0,1.7715536117553712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,1,128,1,bfloat16,fp8,0,1.677529525756836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,1,128,1,fp8,fp8,0,1.6595823287963867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,2,128,1,bfloat16,fp8,0,1.6025632858276366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,2,128,1,bfloat16,bfloat16,0,1.7185327529907226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,8,128,1,bfloat16,bfloat16,0,0.8913071632385254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,4,128,1,bfloat16,fp8,0,1.5886719703674317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,4,128,1,bfloat16,bfloat16,0,1.7479711532592774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,8,128,1,bfloat16,fp8,0,0.8281984329223633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,8,128,1,fp8,fp8,0,0.8941871643066406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,4,128,1,fp8,fp8,0,1.7614463806152343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,1,128,1,bfloat16,bfloat16,0,0.8815983772277832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,1,128,1,fp8,fp8,0,0.7803296089172364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,1,128,1,bfloat16,fp8,0,0.8642864227294922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,2,128,1,bfloat16,bfloat16,0,0.9787823677062988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,2,128,1,bfloat16,fp8,0,0.7901616096496582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,2,128,1,fp8,fp8,0,0.8275055885314941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,4,128,1,bfloat16,bfloat16,0,0.8746879577636719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,4,128,1,bfloat16,fp8,0,0.9242608070373535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,8,4,128,1,fp8,fp8,0,0.7980063915252685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,8,2,128,1,fp8,fp8,0,1.7281503677368164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,1,128,1,fp8,fp8,0,3.526870346069336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,1,128,1,bfloat16,fp8,0,3.7301361083984377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,1,128,1,bfloat16,bfloat16,0,4.337263870239258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,2,128,1,bfloat16,fp8,0,3.6999137878417967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,2,128,1,fp8,fp8,0,3.6263263702392576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,2,128,1,bfloat16,bfloat16,0,4.379294586181641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,4,128,1,bfloat16,bfloat16,0,4.489265441894531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,8,128,1,bfloat16,bfloat16,0,2.172742462158203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,8,128,1,bfloat16,fp8,0,1.9711616516113282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,8,128,1,fp8,fp8,0,1.8890672683715821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,4,128,1,bfloat16,fp8,0,3.6396366119384767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,1,128,1,bfloat16,bfloat16,0,1.9257392883300781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,1,128,1,bfloat16,fp8,0,1.9203088760375977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,1,128,1,fp8,fp8,0,2.0540639877319338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,8,4,128,1,fp8,fp8,0,3.616921615600586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,2,128,1,bfloat16,fp8,0,1.7801055908203125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,2,128,1,fp8,fp8,0,1.7854192733764649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,2,128,1,bfloat16,bfloat16,0,2.2069135665893556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,8,128,1,bfloat16,bfloat16,0,1.0854127883911133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,4,128,1,bfloat16,bfloat16,0,2.0618623733520507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,8,128,1,bfloat16,fp8,0,0.9478768348693848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,4,128,1,bfloat16,fp8,0,2.0485279083251955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,8,4,128,1,fp8,fp8,0,1.8296335220336915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,8,128,1,fp8,fp8,0,0.9466768264770508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,1,128,1,bfloat16,bfloat16,0,0.9667167663574219
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,2,128,1,bfloat16,fp8,0,0.8958527565002441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,1,128,1,fp8,fp8,0,0.9511119842529296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,1,128,1,bfloat16,fp8,0,1.0439439773559571
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,2,128,1,bfloat16,bfloat16,0,1.0752511978149415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,4,128,1,bfloat16,bfloat16,0,1.0327391624450684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,4,128,1,bfloat16,fp8,0,0.913270378112793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,8,128,1,bfloat16,bfloat16,0,0.5629888057708741
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,8,128,1,bfloat16,fp8,0,0.5006063938140869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,8,128,1,fp8,fp8,0,0.5379648208618164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,4,128,1,fp8,fp8,0,0.9810655593872071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,1,128,1,bfloat16,bfloat16,0,0.5319983959197998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,1,128,1,bfloat16,fp8,0,0.5534192085266113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,1,128,1,fp8,fp8,0,0.4492176055908203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,2,128,1,bfloat16,fp8,0,0.4684751987457275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,2,128,1,bfloat16,bfloat16,0,0.486897611618042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,2,128,1,fp8,fp8,0,0.4946239948272705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,4,128,1,bfloat16,bfloat16,0,0.5369935989379883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,4,128,1,bfloat16,fp8,0,0.5123744010925293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,8,4,128,1,fp8,fp8,0,0.535646390914917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,1,128,1,bfloat16,fp8,0,2.5234912872314452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,8,2,128,1,fp8,fp8,0,0.9925392150878907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,1,128,1,fp8,fp8,0,2.4904111862182616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,2,128,1,bfloat16,fp8,0,2.5784591674804687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,1,128,1,bfloat16,bfloat16,0,2.9739856719970703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,2,128,1,fp8,fp8,0,2.681697654724121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,2,128,1,bfloat16,bfloat16,0,3.040300750732422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,8,128,1,bfloat16,bfloat16,0,1.5017135620117188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,8,128,1,fp8,fp8,0,1.369371223449707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,8,128,1,bfloat16,fp8,0,1.5668736457824708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,1,128,1,bfloat16,bfloat16,0,1.5992431640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,4,128,1,bfloat16,fp8,0,2.5573152542114257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,4,128,1,fp8,fp8,0,2.549153518676758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,8,4,128,1,bfloat16,bfloat16,0,3.2025856018066405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,1,128,1,fp8,fp8,0,1.2777024269104005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,1,128,1,bfloat16,fp8,0,1.3650079727172852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,2,128,1,bfloat16,bfloat16,0,1.4951727867126465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,2,128,1,bfloat16,fp8,0,1.5855839729309082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,2,128,1,fp8,fp8,0,1.2752832412719726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,8,128,1,bfloat16,bfloat16,0,0.7936624050140381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,4,128,1,bfloat16,bfloat16,0,1.4441871643066406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,8,128,1,bfloat16,fp8,0,0.8479215621948242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,8,128,1,fp8,fp8,0,0.696504020690918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,4,128,1,fp8,fp8,0,1.346395206451416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,1,128,1,bfloat16,fp8,0,0.6345615863800049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,1,128,1,bfloat16,bfloat16,0,0.710310411453247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,1,128,1,fp8,fp8,0,0.6788047790527344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,2,128,1,bfloat16,bfloat16,0,0.7627647876739502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,2,128,1,bfloat16,fp8,0,0.7688576221466065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,2,128,1,fp8,fp8,0,0.648035192489624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,8,128,1,bfloat16,bfloat16,0,0.42423200607299805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,8,128,1,bfloat16,fp8,0,0.41602401733398436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,4,128,1,bfloat16,bfloat16,0,0.7801616191864014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,4,128,1,fp8,fp8,0,0.718828821182251
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,8,128,1,fp8,fp8,0,0.43108482360839845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,1,128,1,bfloat16,bfloat16,0,0.35091519355773926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,1,128,1,bfloat16,fp8,0,0.3369584083557129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,2,128,1,bfloat16,fp8,0,0.3398240089416504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,2,128,1,bfloat16,bfloat16,0,0.3890655994415283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,2,128,1,fp8,fp8,0,0.3992511987686157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,4,128,1,bfloat16,bfloat16,0,0.3746975898742676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,4,128,1,bfloat16,fp8,0,0.34723520278930664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,4,128,1,fp8,fp8,0,0.3394144058227539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,8,4,128,1,bfloat16,fp8,0,1.68646240234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,8,4,128,1,bfloat16,fp8,0,0.6507423877716064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,1,128,1,bfloat16,fp8,0,3.246460723876953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,1,128,1,fp8,fp8,0,3.3024784088134767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,8,1,128,1,fp8,fp8,0,0.38852639198303224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,2,128,1,bfloat16,fp8,0,3.260465621948242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,2,128,1,bfloat16,bfloat16,0,3.809632110595703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,1,128,1,bfloat16,bfloat16,0,4.15382080078125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,8,128,1,bfloat16,fp8,0,1.7645536422729493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,8,128,1,bfloat16,bfloat16,0,1.9751136779785157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,2,128,1,fp8,fp8,0,3.2819183349609373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,8,128,1,fp8,fp8,0,1.8958448410034179
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,1,128,1,bfloat16,bfloat16,0,1.9735536575317383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,4,128,1,bfloat16,bfloat16,0,3.834489440917969
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,4,128,1,bfloat16,fp8,0,3.3430927276611326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,8,4,128,1,fp8,fp8,0,3.3492752075195313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,1,128,1,bfloat16,fp8,0,1.633950424194336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,1,128,1,fp8,fp8,0,1.8403104782104491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,2,128,1,bfloat16,bfloat16,0,1.8433551788330078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,2,128,1,fp8,fp8,0,1.850539207458496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,8,128,1,bfloat16,bfloat16,0,0.9679375648498535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,8,128,1,bfloat16,fp8,0,0.903923225402832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,4,128,1,bfloat16,fp8,0,1.671615982055664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,4,128,1,fp8,fp8,0,1.674991989135742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,8,128,1,fp8,fp8,0,0.949345588684082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,1,128,1,bfloat16,bfloat16,0,0.9061871528625488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,4,128,1,bfloat16,bfloat16,0,2.0141616821289063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,1,128,1,bfloat16,fp8,0,0.8739952087402344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,1,128,1,fp8,fp8,0,0.8382559776306152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,2,128,1,bfloat16,fp8,0,0.8253359794616699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,2,128,1,bfloat16,bfloat16,0,0.8942895889282226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,2,128,1,fp8,fp8,0,0.8815216064453125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,4,128,1,bfloat16,bfloat16,0,0.9651599884033203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,4,128,1,bfloat16,fp8,0,0.907595157623291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,8,128,1,bfloat16,bfloat16,0,0.49401278495788575
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,8,128,1,bfloat16,fp8,0,0.44394240379333494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,8,4,128,1,fp8,fp8,0,0.8824480056762696
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,1,128,1,bfloat16,bfloat16,0,0.45045762062072753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,1,128,1,fp8,fp8,0,0.4307919979095459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,2,128,1,bfloat16,bfloat16,0,0.4585360050201416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,2,128,1,bfloat16,fp8,0,0.41930561065673827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,2,128,1,fp8,fp8,0,0.43618240356445315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,4,128,1,bfloat16,bfloat16,0,0.4712672233581543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,8,128,1,bfloat16,bfloat16,0,0.24619998931884765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,4,128,1,bfloat16,fp8,0,0.43338398933410643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,8,128,1,bfloat16,fp8,0,0.23703999519348146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,8,128,1,fp8,fp8,0,0.23721599578857422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,4,128,1,fp8,fp8,0,0.4519008159637451
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,1,128,1,bfloat16,fp8,0,0.2111743927001953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,1,128,1,fp8,fp8,0,0.21766080856323242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,8,2,128,1,bfloat16,fp8,0,1.678593635559082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,2,128,1,bfloat16,bfloat16,0,0.23458399772644042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,2,128,1,bfloat16,fp8,0,0.22443039417266847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,2,128,1,fp8,fp8,0,0.21146559715270996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,4,128,1,bfloat16,bfloat16,0,0.23934719562530518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,4,128,1,bfloat16,fp8,0,0.21712799072265626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,4,128,1,fp8,fp8,0,0.22639679908752441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,1,128,1,fp8,fp8,0,1.8882160186767578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,1,128,1,bfloat16,fp8,0,1.9455135345458985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,1,128,1,bfloat16,bfloat16,0,2.1469903945922852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,8,128,1,fp8,fp8,0,0.44225277900695803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,2,128,1,bfloat16,bfloat16,0,2.0947999954223633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,8,1,128,1,bfloat16,fp8,0,0.4603151798248291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,8,1,128,1,bfloat16,bfloat16,0,0.2250063896179199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,8,128,1,bfloat16,bfloat16,0,1.1844063758850099
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,2,128,1,bfloat16,fp8,0,1.8939056396484375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,2,128,1,fp8,fp8,0,1.926598358154297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,4,128,1,bfloat16,fp8,0,1.9577856063842773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,8,128,1,bfloat16,fp8,0,1.0374208450317384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,4,128,1,fp8,fp8,0,2.1453136444091796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,8,4,128,1,bfloat16,bfloat16,0,2.316521644592285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,8,128,1,fp8,fp8,0,1.057587242126465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,1,128,1,bfloat16,bfloat16,0,1.0245519638061524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,1,128,1,bfloat16,fp8,0,0.9675423622131347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,1,128,1,fp8,fp8,0,0.96878080368042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,2,128,1,bfloat16,fp8,0,0.9629343986511231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,2,128,1,bfloat16,bfloat16,0,1.0361040115356446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,2,128,1,fp8,fp8,0,1.0128687858581542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,8,128,1,bfloat16,bfloat16,0,0.5952032089233399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,4,128,1,bfloat16,bfloat16,0,1.086300754547119
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,8,128,1,bfloat16,fp8,0,0.5311024188995361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,4,128,1,bfloat16,fp8,0,1.015017604827881
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,8,128,1,fp8,fp8,0,0.5206783771514892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,1,128,1,bfloat16,bfloat16,0,0.5114048004150391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,1,128,1,bfloat16,fp8,0,0.488483190536499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,8,4,128,1,fp8,fp8,0,1.0523360252380372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,1,128,1,fp8,fp8,0,0.49084959030151365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,2,128,1,bfloat16,fp8,0,0.4903247833251953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,2,128,1,bfloat16,bfloat16,0,0.5418655872344971
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,2,128,1,fp8,fp8,0,0.49759840965270996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,8,128,1,bfloat16,bfloat16,0,0.3220000028610229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,8,128,1,bfloat16,fp8,0,0.2746000051498413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,4,128,1,bfloat16,bfloat16,0,0.5448512077331543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,4,128,1,bfloat16,fp8,0,0.5427807807922364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,8,128,1,fp8,fp8,0,0.274783992767334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,1,128,1,bfloat16,fp8,0,0.2522111892700195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,1,128,1,fp8,fp8,0,0.25527679920196533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,2,128,1,bfloat16,fp8,0,0.24742560386657714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,2,128,1,fp8,fp8,0,0.2602576017379761
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,4,128,1,bfloat16,bfloat16,0,0.28227040767669676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,4,128,1,bfloat16,fp8,0,0.2595232009887695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,4,128,1,fp8,fp8,0,0.26231040954589846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,8,128,1,bfloat16,fp8,0,0.1501744031906128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,8,128,1,bfloat16,bfloat16,0,0.17125600576400757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,8,128,1,fp8,fp8,0,0.14570399522781372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,1,128,1,bfloat16,bfloat16,0,0.1457200050354004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,1,128,1,bfloat16,fp8,0,0.13850879669189453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,1,128,1,fp8,fp8,0,0.13888959884643554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,2,128,1,bfloat16,bfloat16,0,0.1439520001411438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,2,128,1,bfloat16,fp8,0,0.13678879737854005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,2,128,1,fp8,fp8,0,0.13650879859924317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,4,128,1,bfloat16,bfloat16,0,0.15296159982681273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,4,128,1,bfloat16,fp8,0,0.14060640335083008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,1,128,1,bfloat16,fp8,0,1.8134464263916015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,1,128,1,bfloat16,bfloat16,0,1.9040447235107423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,1,128,1,fp8,fp8,0,1.8108896255493163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,2,128,1,bfloat16,bfloat16,0,2.0050527572631838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,8,4,128,1,fp8,fp8,0,0.49807038307189944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,1,128,1,bfloat16,bfloat16,0,0.27467041015625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,8,2,128,1,bfloat16,bfloat16,0,0.26439359188079836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,2,128,1,bfloat16,fp8,0,1.8193344116210937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,8,4,128,1,fp8,fp8,0,0.1351807951927185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,2,128,1,fp8,fp8,0,1.9418991088867188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,8,128,1,bfloat16,bfloat16,0,1.1286144256591797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,4,128,1,bfloat16,bfloat16,0,2.033198356628418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,8,128,1,bfloat16,fp8,0,0.9879008293151855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,4,128,1,bfloat16,fp8,0,1.9954784393310547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,8,4,128,1,fp8,fp8,0,1.8565296173095702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,1,128,1,bfloat16,bfloat16,0,0.9580783843994141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,8,128,1,fp8,fp8,0,0.9874799728393555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,1,128,1,bfloat16,fp8,0,0.9198127746582031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,1,128,1,fp8,fp8,0,0.9737903594970703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,2,128,1,bfloat16,bfloat16,0,1.0391056060791015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,2,128,1,fp8,fp8,0,0.92598876953125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,8,128,1,bfloat16,bfloat16,0,0.5645023822784424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,8,128,1,bfloat16,fp8,0,0.5352767944335938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,8,128,1,fp8,fp8,0,0.5446176052093505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,1,128,1,bfloat16,bfloat16,0,0.4865583896636963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,4,128,1,fp8,fp8,0,0.9360655784606934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,4,128,1,bfloat16,fp8,0,0.9732336044311524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,1,128,1,bfloat16,fp8,0,0.47362241744995115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,1,128,1,fp8,fp8,0,0.47051520347595216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,2,128,1,bfloat16,bfloat16,0,0.5141232013702393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,2,128,1,bfloat16,fp8,0,0.46999521255493165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,2,128,1,fp8,fp8,0,0.4575183868408203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,4,128,1,bfloat16,bfloat16,0,0.5236127853393555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,8,128,1,bfloat16,bfloat16,0,0.2921695947647095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,4,128,1,bfloat16,fp8,0,0.4877967834472656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,8,128,1,bfloat16,fp8,0,0.2572144031524658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,8,4,128,1,fp8,fp8,0,0.4868159770965576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,1,128,1,bfloat16,bfloat16,0,0.25011839866638186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,1,128,1,bfloat16,fp8,0,0.2428447961807251
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,1,128,1,fp8,fp8,0,0.23996479511260987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,2,128,1,bfloat16,fp8,0,0.24200479984283446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,2,128,1,fp8,fp8,0,0.2430704116821289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,4,128,1,bfloat16,bfloat16,0,0.2657360076904297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,4,128,1,bfloat16,fp8,0,0.24768319129943847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,4,128,1,fp8,fp8,0,0.25026400089263917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,8,128,1,bfloat16,bfloat16,0,0.15124959945678712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,8,128,1,bfloat16,fp8,0,0.13747999668121338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,8,128,1,fp8,fp8,0,0.1380560040473938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,1,128,1,bfloat16,bfloat16,0,0.1289072036743164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,1,128,1,bfloat16,fp8,0,0.12065119743347168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,1,128,1,fp8,fp8,0,0.12085280418395997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,2,128,1,bfloat16,bfloat16,0,0.13204480409622193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,2,128,1,bfloat16,fp8,0,0.12211999893188477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,2,128,1,fp8,fp8,0,0.12150399684906006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,4,128,1,bfloat16,bfloat16,0,0.1377120018005371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,4,128,1,bfloat16,fp8,0,0.12753119468688964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,8,4,128,1,fp8,fp8,0,0.12791039943695068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,8,128,1,bfloat16,bfloat16,0,0.07812640070915222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,8,128,1,bfloat16,fp8,0,0.06781280040740967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,8,128,1,fp8,fp8,0,0.07037119865417481
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,1,128,1,bfloat16,bfloat16,0,0.06620000004768371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,2,128,1,bfloat16,fp8,0,0.9004816055297852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,1,128,1,bfloat16,fp8,0,0.06319040060043335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,1,128,1,fp8,fp8,0,0.0645792007446289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,2,128,1,bfloat16,bfloat16,0,0.06653919816017151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,2,128,1,bfloat16,fp8,0,0.06496480107307434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,2,128,1,fp8,fp8,0,0.063128000497818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,4,128,1,bfloat16,fp8,0,0.06636800169944763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,4,128,1,fp8,fp8,0,0.0670144021511078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,8,4,128,1,bfloat16,bfloat16,0,1.032694435119629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,1,128,1,bfloat16,fp8,0,1.0842927932739257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,1,128,1,bfloat16,bfloat16,0,1.1248576164245605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,1,128,1,fp8,fp8,0,1.087831974029541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,8,128,1,fp8,fp8,0,0.2575743913650513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,2,128,1,bfloat16,bfloat16,0,1.1549695968627929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,8,2,128,1,bfloat16,bfloat16,0,0.24945919513702391
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,2,128,1,bfloat16,fp8,0,1.0900527954101562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,2,128,1,fp8,fp8,0,1.1153216361999512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,4,128,1,bfloat16,bfloat16,0,1.2295920372009277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,8,128,1,bfloat16,bfloat16,0,0.6971007823944092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,4,128,1,bfloat16,fp8,0,1.1412079811096192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,8,4,128,1,fp8,fp8,0,1.1220303535461427
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,8,128,1,bfloat16,fp8,0,0.6172319889068604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,8,128,1,fp8,fp8,0,0.6087840080261231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,1,128,1,bfloat16,fp8,0,0.5507552146911621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,1,128,1,bfloat16,bfloat16,0,0.5893280029296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,1,128,1,fp8,fp8,0,0.5494463920593262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,2,128,1,bfloat16,bfloat16,0,0.58362717628479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,8,4,128,1,bfloat16,bfloat16,0,0.06972320079803467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,2,128,1,bfloat16,fp8,0,0.5533840179443359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,2,128,1,fp8,fp8,0,0.5610688209533692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,8,128,1,bfloat16,bfloat16,0,0.3601376056671143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,4,128,1,bfloat16,bfloat16,0,0.6193840026855468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,4,128,1,fp8,fp8,0,0.5774640083312989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,8,128,1,bfloat16,fp8,0,0.3124912023544312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,8,128,1,fp8,fp8,0,0.3155055999755859
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,1,128,1,bfloat16,bfloat16,0,0.290500807762146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,1,128,1,bfloat16,fp8,0,0.28760480880737305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,1,128,1,fp8,fp8,0,0.28147358894348146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,2,128,1,bfloat16,bfloat16,0,0.30035998821258547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,2,128,1,fp8,fp8,0,0.2842400074005127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,4,128,1,bfloat16,bfloat16,0,0.31722559928894045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,8,128,1,bfloat16,bfloat16,0,0.19178080558776855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,8,128,1,bfloat16,fp8,0,0.16379519701004028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,4,128,1,bfloat16,fp8,0,0.30163040161132815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,8,128,1,fp8,fp8,0,0.16255680322647095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,4,128,1,fp8,fp8,0,0.295248007774353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,1,128,1,bfloat16,bfloat16,0,0.15353920459747314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,1,128,1,bfloat16,fp8,0,0.14828480482101442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,1,128,1,fp8,fp8,0,0.15095200538635253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,2,128,1,bfloat16,bfloat16,0,0.1556256055831909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,2,128,1,fp8,fp8,0,0.14833760261535645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,4,128,1,bfloat16,bfloat16,0,0.16469919681549072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,4,128,1,bfloat16,fp8,0,0.1565359950065613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,8,128,1,bfloat16,bfloat16,0,0.10267519950866699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,8,128,1,bfloat16,fp8,0,0.08961439728736878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,8,128,1,fp8,fp8,0,0.09012479782104492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,1,128,1,bfloat16,bfloat16,0,0.08245440125465393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,1,128,1,bfloat16,fp8,0,0.07951520085334778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,1,128,1,fp8,fp8,0,0.07950080037117005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,2,128,1,bfloat16,bfloat16,0,0.0852735996246338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,2,128,1,bfloat16,fp8,0,0.07921760082244873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,2,128,1,fp8,fp8,0,0.0790336012840271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,4,128,1,bfloat16,bfloat16,0,0.09158239960670471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,4,128,1,fp8,fp8,0,0.08026559948921204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,8,128,1,bfloat16,bfloat16,0,0.059006398916244505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,8,128,1,bfloat16,fp8,0,0.051772797107696535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,8,128,1,fp8,fp8,0,0.05180799961090088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,1,128,1,bfloat16,bfloat16,0,0.0503600001335144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,1,128,1,fp8,fp8,0,0.050102400779724124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,2,128,1,bfloat16,bfloat16,0,0.0507423996925354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,2,128,1,bfloat16,fp8,0,0.05034400224685669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,2,128,1,fp8,fp8,0,0.04996480047702789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,4,128,1,bfloat16,bfloat16,0,0.05199679732322693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,4,128,1,bfloat16,fp8,0,0.05023679733276367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,4,128,1,fp8,fp8,0,0.05074399709701538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,8,4,128,1,bfloat16,fp8,0,0.5940991878509522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,1,128,1,bfloat16,bfloat16,0,1.0928288459777833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,8,2,128,1,bfloat16,fp8,0,0.2810192108154297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,2,128,1,bfloat16,fp8,0,0.14795520305633544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,1,128,1,fp8,fp8,0,1.052454376220703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,8,4,128,1,fp8,fp8,0,0.15363680124282836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,2,128,1,bfloat16,fp8,0,1.0681407928466797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,8,4,128,1,bfloat16,fp8,0,0.08020960092544556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,2,128,1,fp8,fp8,0,1.059823989868164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,8,1,128,1,bfloat16,fp8,0,0.04963679909706116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,4,128,1,bfloat16,bfloat16,0,1.2396816253662108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,4,128,1,bfloat16,fp8,0,1.1325311660766602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,8,128,1,bfloat16,bfloat16,0,0.715502405166626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,8,128,1,bfloat16,fp8,0,0.6250912189483643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,4,128,1,fp8,fp8,0,1.147979164123535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,8,128,1,fp8,fp8,0,0.6333407878875732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,1,128,1,bfloat16,bfloat16,0,0.5480544090270996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,1,128,1,fp8,fp8,0,0.5338352203369141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,1,128,1,bfloat16,fp8,0,0.5427599906921386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,2,128,1,bfloat16,bfloat16,0,0.5746399879455566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,1,128,1,bfloat16,fp8,0,1.070622444152832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,2,128,1,bfloat16,fp8,0,0.5426496028900146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,2,128,1,fp8,fp8,0,0.5363264083862305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,8,128,1,bfloat16,bfloat16,0,0.36802399158477783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,4,128,1,bfloat16,fp8,0,0.5788640022277832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,8,128,1,fp8,fp8,0,0.31869120597839357
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,4,128,1,bfloat16,bfloat16,0,0.6178880214691163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,8,4,128,1,fp8,fp8,0,0.5781760215759277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,1,128,1,bfloat16,bfloat16,0,0.28553760051727295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,1,128,1,fp8,fp8,0,0.27294719219207764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,1,128,1,bfloat16,fp8,0,0.2787456035614014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,2,128,1,bfloat16,bfloat16,0,0.2912400007247925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,8,2,128,1,bfloat16,bfloat16,0,1.146463966369629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,2,128,1,fp8,fp8,0,0.2788288116455078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,4,128,1,bfloat16,bfloat16,0,0.3164448022842407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,4,128,1,bfloat16,fp8,0,0.2965008020401001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,4,128,1,fp8,fp8,0,0.2910815954208374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,8,128,1,bfloat16,fp8,0,0.16812800168991088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,8,128,1,bfloat16,bfloat16,0,0.19231200218200684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,8,128,1,fp8,fp8,0,0.1657248020172119
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,1,128,1,bfloat16,bfloat16,0,0.1490880012512207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,1,128,1,fp8,fp8,0,0.14496639966964722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,2,128,1,bfloat16,bfloat16,0,0.1562432050704956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,2,128,1,bfloat16,fp8,0,0.1436895966529846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,2,128,1,fp8,fp8,0,0.14688639640808104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,4,128,1,bfloat16,bfloat16,0,0.16800639629364014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,4,128,1,bfloat16,fp8,0,0.15527679920196533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,4,128,1,fp8,fp8,0,0.15328480005264283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,8,128,1,bfloat16,bfloat16,0,0.10106719732284546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,8,128,1,bfloat16,fp8,0,0.09034559726715088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,8,128,1,fp8,fp8,0,0.09039199948310853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,1,128,1,bfloat16,bfloat16,0,0.07946720123291015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,1,128,1,bfloat16,fp8,0,0.07585440278053283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,1,128,1,fp8,fp8,0,0.0760479986667633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,2,128,1,bfloat16,bfloat16,0,0.0831488013267517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,2,128,1,bfloat16,fp8,0,0.07571840286254883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,2,128,1,fp8,fp8,0,0.07680320143699645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,4,128,1,bfloat16,bfloat16,0,0.0893999993801117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,4,128,1,fp8,fp8,0,0.0815999984741211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,8,128,1,bfloat16,bfloat16,0,0.0545199990272522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,8,128,1,bfloat16,fp8,0,0.046158400177955625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,8,128,1,fp8,fp8,0,0.04676159918308258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,1,128,1,bfloat16,bfloat16,0,0.04241119921207428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,1,128,1,bfloat16,fp8,0,0.04199999868869782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,1,128,1,fp8,fp8,0,0.041812801361083986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,2,128,1,bfloat16,bfloat16,0,0.042659199237823485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,2,128,1,bfloat16,fp8,0,0.04185599982738495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,2,128,1,fp8,fp8,0,0.042131200432777405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,4,128,1,bfloat16,bfloat16,0,0.04648480117321015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,4,128,1,bfloat16,fp8,0,0.04256319999694824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,8,4,128,1,fp8,fp8,0,0.042561599612236024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,8,128,1,bfloat16,bfloat16,0,0.03916159868240356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,8,128,1,bfloat16,fp8,0,0.036929601430892946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,8,128,1,fp8,fp8,0,0.03742400109767914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,1,128,1,bfloat16,bfloat16,0,0.036873599886894225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,1,128,1,bfloat16,fp8,0,0.03593280017375946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,1,128,1,fp8,fp8,0,0.03596000075340271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,2,128,1,bfloat16,bfloat16,0,0.03718560039997101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,2,128,1,bfloat16,fp8,0,0.03601439893245697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,2,128,1,fp8,fp8,0,0.03639839887619019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,4,128,1,bfloat16,bfloat16,0,0.03795360028743744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,4,128,1,bfloat16,fp8,0,0.036313599348068236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,8,128,1,bfloat16,fp8,0,0.3289232015609741
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,1,128,1,bfloat16,bfloat16,0,0.7528143882751465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,8,2,128,1,bfloat16,fp8,0,0.2803584098815918
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,1,128,1,bfloat16,fp8,0,0.7338223934173584
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,1,128,1,fp8,fp8,0,0.7390672206878662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,2,128,1,bfloat16,bfloat16,0,0.7800447940826416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,8,1,128,1,bfloat16,fp8,0,0.14281760454177855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,2,128,1,fp8,fp8,0,0.7348447799682617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,4,128,1,bfloat16,fp8,0,0.7946400165557861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,4,128,1,bfloat16,bfloat16,0,0.8515199661254883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,8,128,1,bfloat16,bfloat16,0,0.5011295795440673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,8,4,128,1,bfloat16,fp8,0,0.08224959969520569
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,4,128,1,fp8,fp8,0,0.7938208103179931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,8,128,1,bfloat16,fp8,0,0.44255361557006834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,1,128,1,bfloat16,fp8,0,0.37137761116027834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,1,128,1,bfloat16,bfloat16,0,0.38517920970916747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,1,128,1,fp8,fp8,0,0.37707200050354006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,2,128,1,bfloat16,bfloat16,0,0.4038640022277832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,2,128,1,bfloat16,fp8,0,0.3762336015701294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,8,4,128,1,fp8,fp8,0,0.036774399876594546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,2,128,1,fp8,fp8,0,0.37590880393981935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,4,128,1,bfloat16,bfloat16,0,0.4347375869750977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,4,128,1,bfloat16,fp8,0,0.4054255962371826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,8,128,1,bfloat16,bfloat16,0,0.2601471900939941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,8,128,1,bfloat16,fp8,0,0.23127200603485107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,8,128,1,fp8,fp8,0,0.22888319492340087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,1,128,1,bfloat16,bfloat16,0,0.20210559368133546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,1,128,1,bfloat16,fp8,0,0.19332640171051024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,1,128,1,fp8,fp8,0,0.1961184024810791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,2,128,1,bfloat16,bfloat16,0,0.20609440803527831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,2,128,1,bfloat16,fp8,0,0.1970352053642273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,2,128,1,fp8,fp8,0,0.1957087993621826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,4,128,1,bfloat16,bfloat16,0,0.22838881015777587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,4,128,1,bfloat16,fp8,0,0.20773921012878419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,8,128,1,bfloat16,bfloat16,0,0.1390560030937195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,8,4,128,1,fp8,fp8,0,0.2098464012145996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,8,128,1,bfloat16,fp8,0,0.12223680019378662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,8,128,1,fp8,fp8,0,0.12114239931106567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,1,128,1,bfloat16,bfloat16,0,0.10794240236282349
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,1,128,1,bfloat16,fp8,0,0.10615520477294922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,1,128,1,fp8,fp8,0,0.10591679811477661
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,2,128,1,bfloat16,bfloat16,0,0.11052000522613525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,2,128,1,bfloat16,fp8,0,0.10720000267028809
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,8,2,128,1,bfloat16,fp8,0,0.7423583984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,2,128,1,fp8,fp8,0,0.10758399963378906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,4,128,1,bfloat16,bfloat16,0,0.12152479887008667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,4,128,1,fp8,fp8,0,0.11241919994354248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,8,128,1,fp8,fp8,0,0.06741920113563538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,8,128,1,bfloat16,bfloat16,0,0.07613279819488525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,1,128,1,bfloat16,bfloat16,0,0.05907359719276428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,1,128,1,bfloat16,fp8,0,0.05760319828987122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,1,128,1,fp8,fp8,0,0.057871997356414795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,2,128,1,bfloat16,bfloat16,0,0.06145600080490112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,2,128,1,fp8,fp8,0,0.05777119994163513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,8,128,1,fp8,fp8,0,0.44788479804992676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,4,128,1,fp8,fp8,0,0.05832800269126892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,4,128,1,bfloat16,fp8,0,0.05910559892654419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,8,128,1,bfloat16,bfloat16,0,0.04238080084323883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,8,128,1,bfloat16,fp8,0,0.03609279990196228
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,8,128,1,fp8,fp8,0,0.035385599732398985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,1,128,1,bfloat16,bfloat16,0,0.03460479974746704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,1,128,1,bfloat16,fp8,0,0.03409120142459869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,1,128,1,fp8,fp8,0,0.03497599959373474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,8,4,128,1,fp8,fp8,0,0.4083087921142578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,2,128,1,bfloat16,bfloat16,0,0.034729599952697754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,2,128,1,bfloat16,fp8,0,0.03470079898834229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,4,128,1,bfloat16,bfloat16,0,0.03601759970188141
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,2,128,1,fp8,fp8,0,0.03481760025024414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,4,128,1,bfloat16,fp8,0,0.035132798552513125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,8,4,128,1,fp8,fp8,0,0.03470079898834229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,8,128,1,bfloat16,bfloat16,0,0.04155679941177368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,8,128,1,bfloat16,fp8,0,0.04045760035514832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,8,128,1,fp8,fp8,0,0.03970080018043518
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,1,128,1,bfloat16,bfloat16,0,0.03431679904460907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,1,128,1,fp8,fp8,0,0.03403359949588776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,2,128,1,bfloat16,bfloat16,0,0.037329599261283875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,2,128,1,bfloat16,fp8,0,0.03688800036907196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,2,128,1,fp8,fp8,0,0.037540799379348753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,4,128,1,bfloat16,bfloat16,0,0.03832319974899292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,4,128,1,bfloat16,fp8,0,0.039680001139640805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,4,128,1,fp8,fp8,0,0.039236798882484436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,8,4,128,1,bfloat16,fp8,0,0.11126240491867065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,8,128,1,bfloat16,fp8,0,0.06645280122756958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,1,128,1,bfloat16,bfloat16,0,0.8014240264892578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,1,128,1,bfloat16,fp8,0,0.7861167907714843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,2,128,1,bfloat16,fp8,0,0.05756319761276245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,8,4,128,1,bfloat16,bfloat16,0,0.06689280271530151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,1,128,1,fp8,fp8,0,0.7831215858459473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,2,128,1,bfloat16,bfloat16,0,0.843289566040039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,2,128,1,bfloat16,fp8,0,0.7952943801879883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,2,128,1,fp8,fp8,0,0.7944831848144531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,4,128,1,bfloat16,bfloat16,0,0.9363535881042481
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,4,128,1,bfloat16,fp8,0,0.8750831604003906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,8,128,1,bfloat16,fp8,0,0.5004559993743897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,8,128,1,bfloat16,bfloat16,0,0.5686016082763672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,8,4,128,1,fp8,fp8,0,0.8606304168701172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,1,128,1,bfloat16,bfloat16,0,0.409881591796875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,8,128,1,fp8,fp8,0,0.498583984375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,1,128,1,bfloat16,fp8,0,0.4079775810241699
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,1,128,1,fp8,fp8,0,0.40758562088012695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,8,1,128,1,bfloat16,fp8,0,0.034236800670623777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,2,128,1,bfloat16,bfloat16,0,0.4313519954681396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,2,128,1,bfloat16,fp8,0,0.4110623836517334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,2,128,1,fp8,fp8,0,0.4057040214538574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,4,128,1,bfloat16,bfloat16,0,0.475600004196167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,4,128,1,bfloat16,fp8,0,0.4417424201965332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,8,128,1,bfloat16,fp8,0,0.25899839401245117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,8,128,1,bfloat16,bfloat16,0,0.2954495906829834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,8,128,1,fp8,fp8,0,0.25830879211425783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,1,128,1,bfloat16,bfloat16,0,0.21606080532073973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,8,4,128,1,fp8,fp8,0,0.4444704055786133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,1,128,1,bfloat16,fp8,0,0.20855200290679932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,1,128,1,fp8,fp8,0,0.21196959018707276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,2,128,1,fp8,fp8,0,0.21546878814697265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,8,128,1,bfloat16,bfloat16,0,0.15463680028915405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,4,128,1,fp8,fp8,0,0.23024001121520996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,4,128,1,bfloat16,fp8,0,0.23132801055908203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,8,128,1,bfloat16,fp8,0,0.13841600418090821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,8,128,1,fp8,fp8,0,0.13826080560684204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,1,128,1,bfloat16,bfloat16,0,0.11410239934921265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,1,128,1,bfloat16,fp8,0,0.1144320011138916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,1,128,1,fp8,fp8,0,0.11473280191421509
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,2,128,1,bfloat16,bfloat16,0,0.12109119892120361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,2,128,1,bfloat16,fp8,0,0.11565760374069214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,2,128,1,fp8,fp8,0,0.11508480310440064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,4,128,1,bfloat16,bfloat16,0,0.13303519487380983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,4,128,1,bfloat16,fp8,0,0.12402880191802979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,8,4,128,1,fp8,fp8,0,0.12390559911727905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,8,128,1,bfloat16,bfloat16,0,0.08442559838294983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,8,128,1,bfloat16,fp8,0,0.07828159928321839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,8,128,1,fp8,fp8,0,0.07880160212516785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,1,128,1,bfloat16,bfloat16,0,0.06684799790382386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,1,128,1,bfloat16,fp8,0,0.06265599727630615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,1,128,1,fp8,fp8,0,0.0640720009803772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,2,128,1,bfloat16,bfloat16,0,0.06898080110549927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,2,128,1,bfloat16,fp8,0,0.06484320163726806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,2,128,1,fp8,fp8,0,0.0639360010623932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,4,128,1,bfloat16,fp8,0,0.0698527991771698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,4,128,1,fp8,fp8,0,0.06975359916687011
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,8,128,1,bfloat16,bfloat16,0,0.050676798820495604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,8,128,1,bfloat16,fp8,0,0.04311360120773315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,8,128,1,fp8,fp8,0,0.04320639967918396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,1,128,1,bfloat16,bfloat16,0,0.0389055997133255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,1,128,1,fp8,fp8,0,0.04041920006275177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,1,128,1,bfloat16,fp8,0,0.04003680050373078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,2,128,1,bfloat16,bfloat16,0,0.0400191992521286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,2,128,1,bfloat16,fp8,0,0.040428799390792844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,2,128,1,fp8,fp8,0,0.04030239880084992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,4,128,1,bfloat16,bfloat16,0,0.0430864006280899
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,4,128,1,bfloat16,fp8,0,0.040987199544906615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,8,128,1,bfloat16,bfloat16,0,0.029692798852920532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,8,128,1,bfloat16,fp8,0,0.029099199175834655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,8,128,1,fp8,fp8,0,0.029388800263404846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,1,128,1,bfloat16,bfloat16,0,0.027105599641799927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,1,128,1,bfloat16,fp8,0,0.027878400683403016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,1,128,1,fp8,fp8,0,0.02770879864692688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,2,128,1,bfloat16,bfloat16,0,0.027377599477767946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,2,128,1,bfloat16,fp8,0,0.02800160050392151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,2,128,1,fp8,fp8,0,0.027796798944473268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,4,128,1,bfloat16,bfloat16,0,0.027988800406455995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,4,128,1,bfloat16,fp8,0,0.0283488005399704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,8,4,128,1,fp8,fp8,0,0.028417599201202393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,8,128,1,bfloat16,bfloat16,0,0.02661119997501373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,8,128,1,bfloat16,fp8,0,0.027463999390602113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,8,128,1,fp8,fp8,0,0.027348798513412476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,1,128,1,bfloat16,fp8,0,0.0270687997341156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,1,128,1,fp8,fp8,0,0.026881599426269533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,2,128,1,bfloat16,bfloat16,0,0.025593599677085875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,2,128,1,bfloat16,fp8,0,0.026716798543930054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,2,128,1,bfloat16,bfloat16,0,0.22539360523223878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,2,128,1,fp8,fp8,0,0.026830399036407472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,4,128,1,bfloat16,bfloat16,0,0.025806400179862975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,4,128,1,bfloat16,bfloat16,0,0.24700961112976075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,4,128,1,bfloat16,fp8,0,0.02709600031375885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,4,128,1,fp8,fp8,0,0.027003198862075806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,1,128,1,bfloat16,bfloat16,0,0.5037983894348145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,1,128,1,bfloat16,fp8,0,0.520307207107544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,1,128,1,fp8,fp8,0,0.520091199874878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,8,4,128,1,bfloat16,bfloat16,0,0.07449600100517273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,2,128,1,bfloat16,bfloat16,0,0.5469359874725341
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,2,128,1,bfloat16,fp8,0,0.5276175975799561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,2,128,1,fp8,fp8,0,0.5268176078796387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,8,4,128,1,fp8,fp8,0,0.041012799739837645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,8,128,1,bfloat16,bfloat16,0,0.4238368034362793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,4,128,1,bfloat16,bfloat16,0,0.643998384475708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,4,128,1,bfloat16,fp8,0,0.5916751861572266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,8,4,128,1,fp8,fp8,0,0.5920767784118652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,8,128,1,bfloat16,fp8,0,0.36642239093780515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,1,128,1,bfloat16,bfloat16,0,0.2606303930282593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,1,128,1,bfloat16,fp8,0,0.26592159271240234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,8,128,1,fp8,fp8,0,0.36561279296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,2,128,1,bfloat16,bfloat16,0,0.28425281047821044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,8,1,128,1,bfloat16,bfloat16,0,0.02556479871273041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,2,128,1,bfloat16,fp8,0,0.2697232007980347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,2,128,1,fp8,fp8,0,0.26965439319610596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,4,128,1,bfloat16,bfloat16,0,0.32899999618530273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,8,128,1,bfloat16,bfloat16,0,0.21989600658416747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,8,128,1,bfloat16,fp8,0,0.1892896056175232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,4,128,1,bfloat16,fp8,0,0.3060208082199097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,8,128,1,fp8,fp8,0,0.19035040140151976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,4,128,1,fp8,fp8,0,0.305401611328125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,8,2,128,1,bfloat16,fp8,0,0.21404318809509276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,1,128,1,bfloat16,bfloat16,0,0.13733439445495604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,1,128,1,bfloat16,fp8,0,0.1409119963645935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,2,128,1,bfloat16,bfloat16,0,0.14891519546508789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,1,128,1,fp8,fp8,0,0.14183839559555053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,2,128,1,fp8,fp8,0,0.14331519603729248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,4,128,1,bfloat16,bfloat16,0,0.17149120569229126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,4,128,1,bfloat16,fp8,0,0.15968480110168456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,4,128,1,fp8,fp8,0,0.1603551983833313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,8,128,1,bfloat16,bfloat16,0,0.11770880222320557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,8,128,1,bfloat16,fp8,0,0.10301599502563477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,1,128,1,bfloat16,bfloat16,0,0.07552319765090942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,1,128,1,bfloat16,fp8,0,0.07763360142707824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,1,128,1,fp8,fp8,0,0.07771999835968017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,2,128,1,bfloat16,bfloat16,0,0.08224639892578126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,2,128,1,bfloat16,fp8,0,0.07903199791908264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,2,128,1,fp8,fp8,0,0.0783407986164093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,4,128,1,bfloat16,bfloat16,0,0.09280160069465637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,4,128,1,bfloat16,fp8,0,0.08732640147209167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,4,128,1,fp8,fp8,0,0.08754720091819763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,8,128,1,bfloat16,bfloat16,0,0.06609119772911072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,8,128,1,bfloat16,fp8,0,0.05793600082397461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,8,128,1,fp8,fp8,0,0.058447998762130735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,1,128,1,bfloat16,fp8,0,0.04359039962291718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,1,128,1,fp8,fp8,0,0.04312480092048645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,2,128,1,bfloat16,bfloat16,0,0.04822559952735901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,2,128,1,bfloat16,fp8,0,0.04441919922828674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,2,128,1,fp8,fp8,0,0.04372960031032562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,4,128,1,bfloat16,bfloat16,0,0.054028797149658206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,4,128,1,bfloat16,fp8,0,0.049481600522994995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,4,128,1,fp8,fp8,0,0.04873439967632294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,8,128,1,bfloat16,bfloat16,0,0.038791999220848083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,8,128,1,bfloat16,fp8,0,0.03245440125465393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,8,128,1,fp8,fp8,0,0.032364800572395325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,1,128,1,bfloat16,bfloat16,0,0.027195200324058533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,1,128,1,bfloat16,fp8,0,0.02861439883708954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,1,128,1,fp8,fp8,0,0.02825759947299957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,2,128,1,bfloat16,bfloat16,0,0.027452799677848815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,2,128,1,bfloat16,fp8,0,0.028832000494003297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,2,128,1,fp8,fp8,0,0.028380799293518066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,4,128,1,bfloat16,bfloat16,0,0.029543998837471008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,4,128,1,bfloat16,fp8,0,0.028652799129486085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,8,4,128,1,fp8,fp8,0,0.028892800211906433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,8,128,1,bfloat16,fp8,0,0.02022400051355362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,8,128,1,fp8,fp8,0,0.02011680006980896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,1,128,1,bfloat16,bfloat16,0,0.017579199373722078
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,1,128,1,bfloat16,fp8,0,0.018855999410152435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,1,128,1,fp8,fp8,0,0.018875199556350707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,2,128,1,bfloat16,bfloat16,0,0.018214400112628936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,2,128,1,bfloat16,fp8,0,0.01905920058488846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,2,128,1,fp8,fp8,0,0.018969599902629853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,4,128,1,bfloat16,bfloat16,0,0.018700799345970152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,4,128,1,bfloat16,fp8,0,0.019329600036144257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,4,128,1,fp8,fp8,0,0.019569599628448488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,8,128,1,bfloat16,bfloat16,0,0.01870400011539459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,8,128,1,bfloat16,fp8,0,0.019487999379634857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,8,128,1,fp8,fp8,0,0.019791999459266664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,1,128,1,bfloat16,bfloat16,0,0.01717280000448227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,8,1,128,1,fp8,fp8,0,0.26571519374847413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,1,128,1,bfloat16,fp8,0,0.018926399946212768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,1,128,1,fp8,fp8,0,0.018481600284576415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,2,128,1,bfloat16,bfloat16,0,0.01743839979171753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,2,128,1,bfloat16,fp8,0,0.019065600633621217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,2,128,1,fp8,fp8,0,0.018688000738620758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,4,128,1,bfloat16,bfloat16,0,0.018006399273872375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,4,128,1,bfloat16,fp8,0,0.01926079988479614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,8,4,128,1,fp8,fp8,0,0.018939200043678283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,8,128,1,bfloat16,bfloat16,0,0.01744000017642975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,8,128,1,bfloat16,fp8,0,0.019281600415706635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,8,128,1,fp8,fp8,0,0.019067199528217317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,1,128,1,bfloat16,bfloat16,0,0.01693760007619858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,1,128,1,bfloat16,fp8,0,0.01878879964351654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,1,128,1,fp8,fp8,0,0.018435199558734894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,2,128,1,bfloat16,bfloat16,0,0.016972799599170686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,2,128,1,bfloat16,fp8,0,0.01876640021800995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,2,128,1,fp8,fp8,0,0.018632000684738158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,4,128,1,bfloat16,bfloat16,0,0.01722400039434433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,4,128,1,bfloat16,fp8,0,0.018838399648666383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,8,4,128,1,fp8,fp8,0,0.018915200233459474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,1,128,1,bfloat16,bfloat16,0,0.21683359146118164
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,1,128,1,bfloat16,fp8,0,0.2266160011291504
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,1,128,1,fp8,fp8,0,0.22462880611419678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,8,8,128,1,fp8,fp8,0,0.1023967981338501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,2,128,1,bfloat16,bfloat16,0,0.2391887903213501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,2,128,1,bfloat16,fp8,0,0.22996959686279297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,4,128,1,bfloat16,bfloat16,0,0.28988959789276125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,8,1,128,1,bfloat16,bfloat16,0,0.04497120082378388
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,4,128,1,bfloat16,fp8,0,0.26386399269104005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,4,128,1,fp8,fp8,0,0.2651952028274536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,8,128,1,bfloat16,bfloat16,0,0.20506079196929933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,8,128,1,bfloat16,fp8,0,0.16312960386276246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,1,128,1,bfloat16,bfloat16,0,0.1166111946105957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,1,128,1,bfloat16,fp8,0,0.12021600008010865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,1,128,1,fp8,fp8,0,0.11975680589675904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,2,128,1,bfloat16,bfloat16,0,0.1280784010887146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,2,128,1,bfloat16,fp8,0,0.12203359603881836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,2,128,1,fp8,fp8,0,0.12156319618225098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,4,128,1,bfloat16,bfloat16,0,0.15371520519256593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,8,8,128,1,bfloat16,bfloat16,0,0.020612800121307374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,4,128,1,bfloat16,fp8,0,0.13940160274505614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,8,128,1,bfloat16,bfloat16,0,0.11106079816818237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,4,128,1,fp8,fp8,0,0.13880159854888915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,8,128,1,bfloat16,fp8,0,0.08893920183181762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,8,128,1,fp8,fp8,0,0.08959839940071106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,1,128,1,bfloat16,bfloat16,0,0.06606400012969971
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,1,128,1,bfloat16,fp8,0,0.06662560105323792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,1,128,1,fp8,fp8,0,0.06668959856033325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,2,128,1,bfloat16,bfloat16,0,0.07122880220413208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,2,128,1,bfloat16,fp8,0,0.06793599724769592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,2,128,1,fp8,fp8,0,0.06770719885826111
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,4,128,1,bfloat16,bfloat16,0,0.08455359935760498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,4,128,1,bfloat16,fp8,0,0.07624639868736267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,8,4,128,1,fp8,fp8,0,0.07626720070838929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,8,128,1,bfloat16,fp8,0,0.050032001733779904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,8,128,1,fp8,fp8,0,0.05049759745597839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,1,128,1,bfloat16,bfloat16,0,0.038129600882530215
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,1,128,1,bfloat16,fp8,0,0.0368912011384964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,1,128,1,fp8,fp8,0,0.036881598830223086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,2,128,1,bfloat16,fp8,0,0.03796960115432739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,2,128,1,fp8,fp8,0,0.03739520013332367
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,4,128,1,bfloat16,bfloat16,0,0.04786239862442017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,4,128,1,bfloat16,fp8,0,0.042961600422859195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,4,128,1,fp8,fp8,0,0.04290719926357269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,8,128,1,bfloat16,bfloat16,0,0.03437120020389557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,8,128,1,bfloat16,fp8,0,0.02571679949760437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,8,128,1,fp8,fp8,0,0.02595199942588806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,1,128,1,bfloat16,bfloat16,0,0.022075200080871583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,1,128,1,bfloat16,fp8,0,0.023340800404548646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,1,128,1,fp8,fp8,0,0.023731200397014617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,2,128,1,bfloat16,bfloat16,0,0.021942399442195892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,2,128,1,bfloat16,fp8,0,0.02362399995326996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,2,128,1,fp8,fp8,0,0.02314240038394928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,4,128,1,bfloat16,bfloat16,0,0.025268799066543578
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,4,128,1,bfloat16,fp8,0,0.023668800294399262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,8,4,128,1,fp8,fp8,0,0.024086399376392363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,8,128,1,bfloat16,bfloat16,0,0.017233599722385407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,8,128,1,bfloat16,fp8,0,0.016899199783802034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,8,128,1,fp8,fp8,0,0.016711999475955964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,1,128,1,bfloat16,bfloat16,0,0.015150399506092071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,1,128,1,bfloat16,fp8,0,0.015427200496196747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,1,128,1,fp8,fp8,0,0.01579200029373169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,2,128,1,bfloat16,bfloat16,0,0.015187199413776397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,8,2,128,1,fp8,fp8,0,0.22818078994750976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,2,128,1,fp8,fp8,0,0.015587200224399567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,4,128,1,bfloat16,bfloat16,0,0.015835200250148774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,4,128,1,bfloat16,fp8,0,0.0160288006067276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,4,128,1,fp8,fp8,0,0.015880000591278077
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,8,128,1,bfloat16,bfloat16,0,0.015769599378108977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,8,128,1,bfloat16,fp8,0,0.015833599865436553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,8,128,1,fp8,fp8,0,0.015943999588489532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,1,128,1,bfloat16,bfloat16,0,0.01436000019311905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,1,128,1,bfloat16,fp8,0,0.015278400480747223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,1,128,1,fp8,fp8,0,0.014905600249767304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,2,128,1,bfloat16,bfloat16,0,0.01475519984960556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,8,8,128,1,fp8,fp8,0,0.1629631996154785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,2,128,1,bfloat16,fp8,0,0.014931200444698334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,2,128,1,fp8,fp8,0,0.015254400670528412
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,4,128,1,bfloat16,bfloat16,0,0.015140800178050995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,4,128,1,bfloat16,fp8,0,0.015275199711322785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,8,4,128,1,fp8,fp8,0,0.015348799526691437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,8,128,1,bfloat16,bfloat16,0,0.014936000108718872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,8,128,1,bfloat16,fp8,0,0.015510399639606477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,8,128,1,fp8,fp8,0,0.01549919992685318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,1,128,1,bfloat16,bfloat16,0,0.014300799369812012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,1,128,1,bfloat16,fp8,0,0.015022400021553039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,1,128,1,fp8,fp8,0,0.015113599598407745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,2,128,1,bfloat16,bfloat16,0,0.014276799559593201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,2,128,1,bfloat16,fp8,0,0.015145599842071533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,2,128,1,fp8,fp8,0,0.015132799744606018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,4,128,1,bfloat16,bfloat16,0,0.014579200744628906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,4,128,1,bfloat16,fp8,0,0.015223999321460725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,8,4,128,1,fp8,fp8,0,0.015508800745010376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,8,128,1,bfloat16,fp8,0,0.015143999457359314
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,8,128,1,fp8,fp8,0,0.015140800178050995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,8,128,1,bfloat16,bfloat16,0,0.014113600552082061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,1,128,1,bfloat16,bfloat16,0,0.013849599659442902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,1,128,1,bfloat16,fp8,0,0.014868800342082978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,1,128,1,fp8,fp8,0,0.014937600493431092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,2,128,1,bfloat16,bfloat16,0,0.013828800618648529
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,2,128,1,bfloat16,fp8,0,0.014793600142002105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,2,128,1,fp8,fp8,0,0.014851200580596923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,4,128,1,bfloat16,bfloat16,0,0.01393280029296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,4,128,1,bfloat16,fp8,0,0.01491519957780838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,8,4,128,1,fp8,fp8,0,0.015048000216484069
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,1,128,1,bfloat16,bfloat16,0,0.11236799955368042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,1,128,1,bfloat16,fp8,0,0.11678400039672851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,1,128,1,fp8,fp8,0,0.11829440593719483
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,2,128,1,bfloat16,bfloat16,0,0.1260848045349121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,2,128,1,bfloat16,bfloat16,0,0.04164479970932007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,2,128,1,bfloat16,fp8,0,0.12282079458236694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,2,128,1,fp8,fp8,0,0.12390400171279907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,4,128,1,bfloat16,bfloat16,0,0.15089279413223267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,4,128,1,bfloat16,fp8,0,0.1394047975540161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,8,128,1,bfloat16,bfloat16,0,0.1115007996559143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,8,128,1,bfloat16,fp8,0,0.08906239867210389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,8,128,1,fp8,fp8,0,0.08874080181121827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,1,128,1,bfloat16,fp8,0,0.064792001247406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,2,128,1,bfloat16,bfloat16,0,0.06931840181350708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,2,128,1,bfloat16,fp8,0,0.06695520281791686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,8,2,128,1,bfloat16,fp8,0,0.14339840412139893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,2,128,1,fp8,fp8,0,0.0665328025817871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,4,128,1,bfloat16,bfloat16,0,0.08307520151138306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,4,128,1,bfloat16,fp8,0,0.07363359928131104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,4,128,1,fp8,fp8,0,0.0740719974040985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,8,128,1,bfloat16,bfloat16,0,0.06161119937896729
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,8,2,128,1,bfloat16,fp8,0,0.01582240015268326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,8,128,1,bfloat16,fp8,0,0.047444799542427064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,8,128,1,fp8,fp8,0,0.04748800098896026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,1,128,1,bfloat16,bfloat16,0,0.03516640067100525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,1,128,1,bfloat16,fp8,0,0.03309440016746521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,1,128,1,fp8,fp8,0,0.032497599720954895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,2,128,1,bfloat16,bfloat16,0,0.039427199959754945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,2,128,1,bfloat16,fp8,0,0.033073601126670835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,2,128,1,fp8,fp8,0,0.03419199883937836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,4,128,1,bfloat16,fp8,0,0.03978559970855713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,4,128,1,bfloat16,bfloat16,0,0.045767998695373534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,8,4,128,1,fp8,fp8,0,0.03920480012893677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,8,128,1,bfloat16,bfloat16,0,0.033025598526000975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,8,128,1,bfloat16,fp8,0,0.02223999947309494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,8,128,1,fp8,fp8,0,0.0230320006608963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,1,128,1,bfloat16,bfloat16,0,0.0191648006439209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,1,128,1,bfloat16,fp8,0,0.020652799308300017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,1,128,1,fp8,fp8,0,0.02056960016489029
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,2,128,1,bfloat16,fp8,0,0.02060000002384186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,2,128,1,fp8,fp8,0,0.020531199872493744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,4,128,1,bfloat16,bfloat16,0,0.022782400250434875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,4,128,1,bfloat16,fp8,0,0.020740799605846405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,4,128,1,fp8,fp8,0,0.02093600034713745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,8,128,1,bfloat16,bfloat16,0,0.016102400422096253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,8,128,1,bfloat16,fp8,0,0.015230399370193482
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,8,128,1,fp8,fp8,0,0.0151296004652977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,1,128,1,bfloat16,bfloat16,0,0.01390720009803772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,1,128,1,bfloat16,fp8,0,0.014270399510860444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,1,128,1,fp8,fp8,0,0.014412799477577209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,2,128,1,bfloat16,fp8,0,0.014351999759674073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,2,128,1,fp8,fp8,0,0.014499199390411378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,4,128,1,bfloat16,bfloat16,0,0.014891199767589569
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,4,128,1,bfloat16,fp8,0,0.014878399670124054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,4,128,1,fp8,fp8,0,0.01483200043439865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,8,128,1,bfloat16,bfloat16,0,0.014287999272346497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,8,128,1,bfloat16,fp8,0,0.014044800400733947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,8,128,1,fp8,fp8,0,0.014060799777507783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,1,128,1,bfloat16,bfloat16,0,0.013220800459384919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,1,128,1,bfloat16,fp8,0,0.013713599741458892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,8,4,128,1,fp8,fp8,0,0.13884960412979125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,1,128,1,fp8,fp8,0,0.013961599767208099
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,2,128,1,bfloat16,bfloat16,0,0.01329279989004135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,1,128,1,bfloat16,bfloat16,0,0.06249439716339111
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,2,128,1,bfloat16,fp8,0,0.013784000277519226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,2,128,1,fp8,fp8,0,0.013790400326251983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,4,128,1,bfloat16,bfloat16,0,0.01364160031080246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,4,128,1,bfloat16,fp8,0,0.013903999328613281
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,8,1,128,1,fp8,fp8,0,0.06469119787216186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,8,4,128,1,fp8,fp8,0,0.013779200613498688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,8,128,1,bfloat16,bfloat16,0,0.013574400544166565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,8,128,1,bfloat16,fp8,0,0.013763199746608733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,8,128,1,fp8,fp8,0,0.013531200587749481
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,1,128,1,bfloat16,fp8,0,0.013646399974822998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,1,128,1,fp8,fp8,0,0.013579200208187103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,2,128,1,bfloat16,bfloat16,0,0.012921600043773651
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,2,128,1,bfloat16,fp8,0,0.013654400408267976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,2,128,1,fp8,fp8,0,0.013488000631332398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,4,128,1,bfloat16,bfloat16,0,0.01318880021572113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,4,128,1,bfloat16,fp8,0,0.013447999954223633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,4,128,1,fp8,fp8,0,0.013463999330997466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,8,128,1,bfloat16,bfloat16,0,0.012756800651550293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,8,128,1,bfloat16,fp8,0,0.013223999738693237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,8,128,1,fp8,fp8,0,0.01329600065946579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,1,128,1,bfloat16,bfloat16,0,0.01239679977297783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,1,128,1,bfloat16,fp8,0,0.01305759996175766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,1,128,1,fp8,fp8,0,0.013097600638866424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,2,128,1,bfloat16,bfloat16,0,0.012328000366687774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,2,128,1,bfloat16,fp8,0,0.013009600341320038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,2,128,1,fp8,fp8,0,0.013075199723243714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,4,128,1,bfloat16,bfloat16,0,0.01260959953069687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,4,128,1,bfloat16,fp8,0,0.01308320015668869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,8,4,128,1,fp8,fp8,0,0.013078400492668152
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,8,128,1,bfloat16,fp8,0,0.012908799946308136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,8,128,1,fp8,fp8,0,0.013020800054073333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,1,128,1,bfloat16,bfloat16,0,0.01213119998574257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,1,128,1,bfloat16,fp8,0,0.013025599718093871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,1,128,1,fp8,fp8,0,0.012851199507713318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,2,128,1,bfloat16,bfloat16,0,0.01236959993839264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,2,128,1,bfloat16,fp8,0,0.012783999741077422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,2,128,1,fp8,fp8,0,0.01279360055923462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,4,128,1,bfloat16,bfloat16,0,0.01223519966006279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,4,128,1,bfloat16,fp8,0,0.012865599989891053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,4,128,1,fp8,fp8,0,0.012894399464130402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,8,2,128,1,bfloat16,bfloat16,0,0.020032000541687012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,1,128,1,bfloat16,bfloat16,0,0.06545119881629943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,1,128,1,bfloat16,fp8,0,0.06574879884719849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,1,128,1,fp8,fp8,0,0.06571840047836304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,2,128,1,bfloat16,bfloat16,0,0.07154080271720886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,2,128,1,bfloat16,fp8,0,0.06733440160751343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,2,128,1,fp8,fp8,0,0.0681168019771576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,4,128,1,bfloat16,bfloat16,0,0.08608800172805786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,4,128,1,bfloat16,fp8,0,0.07434719800949097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,8,4,128,1,fp8,fp8,0,0.07507680058479309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,8,128,1,bfloat16,bfloat16,0,0.07323520183563233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,8,2,128,1,bfloat16,bfloat16,0,0.01398559957742691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,8,128,1,bfloat16,fp8,0,0.06541119813919068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,8,128,1,fp8,fp8,0,0.06520959734916687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,1,128,1,bfloat16,bfloat16,0,0.036950400471687316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,2,128,1,bfloat16,bfloat16,0,0.039982399344444274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,1,128,1,bfloat16,fp8,0,0.0334991991519928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,2,128,1,bfloat16,fp8,0,0.03475840091705322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,4,128,1,bfloat16,bfloat16,0,0.04667679965496063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,4,128,1,bfloat16,fp8,0,0.040275201201438904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,4,128,1,fp8,fp8,0,0.04077439904212952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,8,128,1,bfloat16,bfloat16,0,0.040078398585319516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,8,128,1,bfloat16,fp8,0,0.03167040050029755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,8,128,1,fp8,fp8,0,0.03195039927959442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,8,8,128,1,bfloat16,bfloat16,0,0.06186400055885315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,1,128,1,bfloat16,fp8,0,0.021160000562667848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,1,128,1,fp8,fp8,0,0.021367999911308288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,2,128,1,bfloat16,fp8,0,0.021107199788093566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,2,128,1,fp8,fp8,0,0.021294400095939636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,2,128,1,bfloat16,bfloat16,0,0.020216000080108643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,4,128,1,bfloat16,bfloat16,0,0.02244960069656372
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,4,128,1,bfloat16,fp8,0,0.02150239944458008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,4,128,1,fp8,fp8,0,0.021411199867725373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,8,128,1,bfloat16,bfloat16,0,0.019673599302768706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,8,128,1,bfloat16,fp8,0,0.019920000433921815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,8,128,1,fp8,fp8,0,0.01968960016965866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,1,128,1,bfloat16,bfloat16,0,0.014620800316333771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,1,128,1,bfloat16,fp8,0,0.014870400726795196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,1,128,1,fp8,fp8,0,0.014883199334144592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,2,128,1,bfloat16,bfloat16,0,0.014424000680446625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,2,128,1,bfloat16,fp8,0,0.014948800206184387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,2,128,1,fp8,fp8,0,0.014630399644374847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,4,128,1,bfloat16,bfloat16,0,0.01523679941892624
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,4,128,1,bfloat16,fp8,0,0.015131199359893798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,8,1,128,1,bfloat16,bfloat16,0,0.012675200402736665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,8,4,128,1,fp8,fp8,0,0.015087999403476715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,8,128,1,bfloat16,bfloat16,0,0.013625599443912506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,8,128,1,bfloat16,fp8,0,0.013982400298118591
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,8,128,1,fp8,fp8,0,0.013891200721263885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,1,128,1,bfloat16,bfloat16,0,0.013142399489879608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,1,128,1,bfloat16,fp8,0,0.013819199800491334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,1,128,1,fp8,fp8,0,0.013665600121021271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,2,128,1,bfloat16,bfloat16,0,0.013172799348831176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,2,128,1,bfloat16,fp8,0,0.013675199449062347
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,2,128,1,fp8,fp8,0,0.01366879940032959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,4,128,1,bfloat16,bfloat16,0,0.013339200615882873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,4,128,1,bfloat16,fp8,0,0.013897599279880523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,8,4,128,1,fp8,fp8,0,0.013760000467300415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,8,128,1,bfloat16,bfloat16,0,0.013031999766826629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,8,128,1,bfloat16,fp8,0,0.013359999656677246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,8,128,1,fp8,fp8,0,0.013526399433612824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,1,128,1,bfloat16,bfloat16,0,0.013027200102806091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,8,8,128,1,bfloat16,bfloat16,0,0.01236959993839264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,1,128,1,fp8,fp8,0,0.013689599931240082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,2,128,1,bfloat16,fp8,0,0.013579200208187103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,2,128,1,fp8,fp8,0,0.013767999410629273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,4,128,1,bfloat16,bfloat16,0,0.013305599987506866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,4,128,1,bfloat16,fp8,0,0.013924799859523773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,4,128,1,fp8,fp8,0,0.013494400680065155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,8,128,1,bfloat16,bfloat16,0,0.01252640038728714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,8,128,1,bfloat16,fp8,0,0.013240000605583191
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,8,128,1,fp8,fp8,0,0.013335999846458436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,1,128,1,bfloat16,bfloat16,0,0.012415999919176102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,1,128,1,bfloat16,fp8,0,0.013022400438785553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,1,128,1,fp8,fp8,0,0.013246400654315949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,2,128,1,bfloat16,bfloat16,0,0.0123648002743721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,2,128,1,bfloat16,fp8,0,0.013038399815559387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,2,128,1,fp8,fp8,0,0.012955200672149659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,4,128,1,bfloat16,bfloat16,0,0.012612800300121307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,4,128,1,bfloat16,fp8,0,0.013166399300098419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,8,4,128,1,fp8,fp8,0,0.013262400031089782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,8,128,1,bfloat16,bfloat16,0,0.012198399752378464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,8,128,1,bfloat16,fp8,0,0.012905600666999816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,8,128,1,fp8,fp8,0,0.012992000579833985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,1,128,1,bfloat16,bfloat16,0,0.01215519979596138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,1,128,1,bfloat16,fp8,0,0.012899200618267059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,1,128,1,fp8,fp8,0,0.012716799974441528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,2,128,1,bfloat16,bfloat16,0,0.01231520026922226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,2,128,1,bfloat16,fp8,0,0.012744000554084778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,2,128,1,fp8,fp8,0,0.012868799269199371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,4,128,1,bfloat16,bfloat16,0,0.012200000137090683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,4,128,1,bfloat16,fp8,0,0.012886400520801543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,8,4,128,1,fp8,fp8,0,0.012734399735927581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,1,128,1,fp8,fp8,0,0.0338128000497818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,8,128,1,bfloat16,fp8,0,0.012556800246238708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,8,128,1,bfloat16,bfloat16,0,0.012095999717712403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,8,128,1,fp8,fp8,0,0.012649600207805634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,1,128,1,bfloat16,bfloat16,0,0.012054400146007537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,8,2,128,1,fp8,fp8,0,0.03444640040397644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,1,128,1,bfloat16,fp8,0,0.012803199887275695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,2,128,1,bfloat16,bfloat16,0,0.012038400024175644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,2,128,1,fp8,fp8,0,0.012700800597667695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,4,128,1,bfloat16,bfloat16,0,0.012055999785661697
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,4,128,1,bfloat16,fp8,0,0.012723200023174286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,4,128,1,fp8,fp8,0,0.012747199833393097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,1,128,1,bfloat16,bfloat16,0,0.03696320056915283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,1,128,1,fp8,fp8,0,0.035831999778747556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,2,128,1,bfloat16,bfloat16,0,0.04185760021209717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,2,128,1,bfloat16,fp8,0,0.0364656001329422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,2,128,1,fp8,fp8,0,0.03646079897880554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,4,128,1,bfloat16,bfloat16,0,0.061715197563171384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,4,128,1,bfloat16,fp8,0,0.05939199924468994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,4,128,1,fp8,fp8,0,0.05944960117340088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,8,128,1,bfloat16,bfloat16,0,0.055390399694442746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,8,128,1,bfloat16,fp8,0,0.050097602605819705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,8,128,1,fp8,fp8,0,0.050102400779724124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,2,128,1,bfloat16,bfloat16,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,1,128,1,bfloat16,bfloat16,0,0.02051199972629547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,1,128,1,bfloat16,fp8,0,0.021905599534511565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,1,128,1,fp8,fp8,0,0.021865600347518922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,2,128,1,bfloat16,bfloat16,0,0.020947200059890748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,2,128,1,bfloat16,fp8,0,0.02168000042438507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,2,128,1,fp8,fp8,0,0.02226720005273819
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,4,128,1,bfloat16,bfloat16,0,0.029897600412368774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,4,128,1,bfloat16,fp8,0,0.03120799958705902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,8,4,128,1,fp8,fp8,0,0.030694401264190672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,8,128,1,bfloat16,bfloat16,0,0.02728799879550934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,8,128,1,bfloat16,fp8,0,0.02877599895000458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,8,128,1,fp8,fp8,0,0.029049599170684816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,1,128,1,bfloat16,bfloat16,0,0.014404800534248353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,1,128,1,bfloat16,fp8,0,0.015489600598812103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,1,128,1,fp8,fp8,0,0.015030400454998016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,2,128,1,bfloat16,bfloat16,0,0.015065599977970124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,2,128,1,bfloat16,fp8,0,0.01518079936504364
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,2,128,1,fp8,fp8,0,0.015145599842071533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,4,128,1,bfloat16,bfloat16,0,0.018628799915313722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,4,128,1,bfloat16,fp8,0,0.019627200067043306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,8,4,128,1,fp8,fp8,0,0.019601599872112276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,8,128,1,bfloat16,bfloat16,0,0.018007999658584593
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,1,128,1,fp8,fp8,0,0.012702399492263794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,8,128,1,bfloat16,fp8,0,0.018731200695037843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,8,2,128,1,bfloat16,fp8,0,0.012542399764060973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,8,128,1,fp8,fp8,0,0.018702399730682374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,1,128,1,bfloat16,bfloat16,0,0.013412800431251527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,8,1,128,1,bfloat16,bfloat16,0,0.019492800533771514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,1,128,1,fp8,fp8,0,0.01422560065984726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,1,128,1,bfloat16,fp8,0,0.014076800644397735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,2,128,1,bfloat16,bfloat16,0,0.013497599959373474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,2,128,1,bfloat16,fp8,0,0.014124800264835358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,8,1,128,1,bfloat16,fp8,0,0.035683199763298035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,2,128,1,fp8,fp8,0,0.014144000411033631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,4,128,1,bfloat16,fp8,0,0.014156800508499146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,4,128,1,fp8,fp8,0,0.014268800616264343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,8,128,1,bfloat16,bfloat16,0,0.013081599771976472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,8,128,1,bfloat16,fp8,0,0.013500800728797913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,8,128,1,fp8,fp8,0,0.013678400218486786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,1,128,1,bfloat16,bfloat16,0,0.012806400656700134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,1,128,1,bfloat16,fp8,0,0.013691200315952301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,2,128,1,bfloat16,fp8,0,0.01385599970817566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,2,128,1,fp8,fp8,0,0.013799999654293061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,4,128,1,bfloat16,bfloat16,0,0.01300159990787506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,8,128,1,bfloat16,bfloat16,0,0.012831999361515046
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,8,128,1,bfloat16,fp8,0,0.013433599472045898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,8,128,1,fp8,fp8,0,0.013424000144004822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,1,128,1,bfloat16,bfloat16,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,1,128,1,bfloat16,fp8,0,0.0130048006772995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,1,128,1,fp8,fp8,0,0.01300159990787506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,2,128,1,bfloat16,bfloat16,0,0.012646399438381195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,2,128,1,bfloat16,fp8,0,0.013195200264453888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,8,1,128,1,bfloat16,fp8,0,0.013471999764442444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,2,128,1,fp8,fp8,0,0.013065600395202636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,4,128,1,bfloat16,fp8,0,0.013288000226020813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,4,128,1,bfloat16,bfloat16,0,0.012323199957609176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,8,4,128,1,fp8,fp8,0,0.013198399543762207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,8,128,1,bfloat16,bfloat16,0,0.012017600238323212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,8,128,1,bfloat16,fp8,0,0.01281919926404953
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,8,128,1,fp8,fp8,0,0.012716799974441528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,1,128,1,bfloat16,bfloat16,0,0.01225920021533966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,1,128,1,bfloat16,fp8,0,0.012732799351215362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,1,128,1,fp8,fp8,0,0.012830400466918945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,2,128,1,bfloat16,bfloat16,0,0.012144000083208085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,2,128,1,bfloat16,fp8,0,0.012891200184822083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,2,128,1,fp8,fp8,0,0.0127920001745224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,4,128,1,bfloat16,bfloat16,0,0.012107200175523757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,4,128,1,bfloat16,fp8,0,0.012665599584579468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,8,4,128,1,fp8,fp8,0,0.01276479959487915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,8,128,1,bfloat16,bfloat16,0,0.011899200081825257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,8,128,1,bfloat16,fp8,0,0.012600000202655792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,8,128,1,fp8,fp8,0,0.01247360035777092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,1,128,1,bfloat16,bfloat16,0,0.012011200189590454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,1,128,1,bfloat16,fp8,0,0.012620800733566284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,1,128,1,fp8,fp8,0,0.012751999497413635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,2,128,1,bfloat16,bfloat16,0,0.011883199959993363
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,2,128,1,bfloat16,fp8,0,0.012619200348854064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,2,128,1,fp8,fp8,0,0.012627199292182922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,4,128,1,bfloat16,bfloat16,0,0.011915200203657151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,4,128,1,bfloat16,fp8,0,0.0124719999730587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,8,4,128,1,fp8,fp8,0,0.012619200348854064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,8,128,1,bfloat16,bfloat16,0,0.011590400338172912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,8,128,1,bfloat16,fp8,0,0.012444800138473511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,8,128,1,fp8,fp8,0,0.012479999661445617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,1,128,1,bfloat16,bfloat16,0,0.01197120025753975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,1,128,1,bfloat16,fp8,0,0.012656000256538392
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,1,128,1,fp8,fp8,0,0.012811200320720672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,2,128,1,bfloat16,bfloat16,0,0.011777599900960922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,2,128,1,bfloat16,fp8,0,0.012702399492263794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,2,128,1,fp8,fp8,0,0.012444800138473511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,4,128,1,bfloat16,bfloat16,0,0.011830399930477142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,4,128,1,bfloat16,fp8,0,0.012374400347471236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,8,4,128,1,fp8,fp8,0,0.012600000202655792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,8,4,128,1,bfloat16,bfloat16,0,0.013532799482345582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,1,128,1,bfloat16,fp8,0,0.024051199853420257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,1,128,1,fp8,fp8,0,0.02479040026664734
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,2,128,1,bfloat16,bfloat16,0,0.02948960065841675
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,2,128,1,bfloat16,bfloat16,0,0.012992000579833985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,2,128,1,bfloat16,fp8,0,0.03298400044441223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,2,128,1,fp8,fp8,0,0.03240959942340851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,4,128,1,bfloat16,fp8,0,0.050704002380371094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,4,128,1,fp8,fp8,0,0.013504000008106231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,4,128,1,fp8,fp8,0,0.050430399179458615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,8,128,1,bfloat16,bfloat16,0,0.04272480010986328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,8,128,1,bfloat16,fp8,0,0.04791040122509003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,8,128,1,fp8,fp8,0,0.04782879948616028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,1,128,1,bfloat16,bfloat16,0,0.01607999950647354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,1,128,1,bfloat16,fp8,0,0.016390399634838106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,1,128,1,fp8,fp8,0,0.016204799711704253
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,2,128,1,bfloat16,bfloat16,0,0.019331200420856474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,2,128,1,bfloat16,fp8,0,0.020576000213623047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,2,128,1,fp8,fp8,0,0.020636799931526183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,4,128,1,bfloat16,bfloat16,0,0.027403199672698976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,4,128,1,bfloat16,fp8,0,0.029684799909591674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,8,4,128,1,fp8,fp8,0,0.02975040078163147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,8,128,1,bfloat16,bfloat16,0,0.025600001215934753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,8,128,1,bfloat16,fp8,0,0.028279998898506166
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,8,128,1,fp8,fp8,0,0.028377598524093627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,1,128,1,bfloat16,bfloat16,0,0.013924799859523773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,1,128,1,bfloat16,fp8,0,0.014737600088119506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,1,128,1,fp8,fp8,0,0.015014399588108063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,2,128,1,bfloat16,bfloat16,0,0.01383039951324463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,2,128,1,bfloat16,fp8,0,0.014694400131702423
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,4,128,1,bfloat16,bfloat16,0,0.018217599391937254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,4,128,1,bfloat16,fp8,0,0.019441600143909454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,4,128,1,fp8,fp8,0,0.019468800723552705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,8,128,1,bfloat16,bfloat16,0,0.0172447994351387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,8,128,1,bfloat16,fp8,0,0.018387199938297273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,8,128,1,fp8,fp8,0,0.018398399651050567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,1,128,1,bfloat16,bfloat16,0,0.013483199477195739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,1,128,1,bfloat16,fp8,0,0.01419840008020401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,1,128,1,fp8,fp8,0,0.014179199934005737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,2,128,1,bfloat16,bfloat16,0,0.013142399489879608
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,2,128,1,bfloat16,fp8,0,0.013803200423717498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,2,128,1,fp8,fp8,0,0.013814400136470794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,4,128,1,bfloat16,bfloat16,0,0.013247999548912048
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,4,128,1,bfloat16,fp8,0,0.013831999897956849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,8,4,128,1,fp8,fp8,0,0.01387999951839447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,8,128,1,bfloat16,bfloat16,0,0.012751999497413635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,8,128,1,bfloat16,fp8,0,0.013369600474834441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,8,128,1,fp8,fp8,0,0.013380800187587739
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,1,128,1,bfloat16,bfloat16,0,0.02234559953212738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,1,128,1,bfloat16,bfloat16,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,1,128,1,bfloat16,fp8,0,0.01324319988489151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,1,128,1,fp8,fp8,0,0.013262400031089782
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,2,128,1,bfloat16,bfloat16,0,0.012174399942159653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,1,128,1,fp8,fp8,0,0.013435199856758118
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,2,128,1,bfloat16,fp8,0,0.013363200426101684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,2,128,1,fp8,fp8,0,0.01324480026960373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,4,128,1,bfloat16,bfloat16,0,0.012600000202655792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,4,128,1,fp8,fp8,0,0.013068799674510957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,8,128,1,bfloat16,bfloat16,0,0.012439999729394913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,8,128,1,bfloat16,fp8,0,0.012948800623416901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,8,128,1,fp8,fp8,0,0.01284320056438446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,1,128,1,bfloat16,bfloat16,0,0.012377600371837615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,1,128,1,bfloat16,fp8,0,0.012934400141239167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,1,128,1,fp8,fp8,0,0.012804800271987915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,2,128,1,bfloat16,bfloat16,0,0.012094400078058242
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,2,128,1,bfloat16,fp8,0,0.012593600153923034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,2,128,1,fp8,fp8,0,0.012625600397586822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,4,128,1,bfloat16,bfloat16,0,0.012332800030708312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,4,128,1,bfloat16,fp8,0,0.012721599638462066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,8,4,128,1,fp8,fp8,0,0.012812800705432892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,8,128,1,bfloat16,bfloat16,0,0.012139199674129486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,8,128,1,bfloat16,fp8,0,0.012579199671745301
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,8,128,1,fp8,fp8,0,0.01265760064125061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,1,128,1,bfloat16,bfloat16,0,0.012028799951076507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,1,128,1,bfloat16,fp8,0,0.012572799623012543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,1,128,1,fp8,fp8,0,0.012782399356365205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,2,128,1,bfloat16,fp8,0,0.012385600060224534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,2,128,1,fp8,fp8,0,0.01249919980764389
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,4,128,1,bfloat16,bfloat16,0,0.011721599847078323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,4,128,1,bfloat16,fp8,0,0.01250080019235611
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,4,128,1,fp8,fp8,0,0.012548799812793731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,8,128,1,bfloat16,bfloat16,0,0.011803200095891952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,8,128,1,bfloat16,fp8,0,0.012392000108957291
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,8,128,1,fp8,fp8,0,0.012468799948692322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,1,128,1,bfloat16,bfloat16,0,0.011897599697113037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,1,128,1,bfloat16,fp8,0,0.012591999769210816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,8,2,128,1,fp8,fp8,0,0.014505599439144135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,1,128,1,fp8,fp8,0,0.012689599394798278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,2,128,1,bfloat16,bfloat16,0,0.011774399876594543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,2,128,1,bfloat16,fp8,0,0.012433599680662155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,4,128,1,bfloat16,bfloat16,0,0.011587200313806533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,4,128,1,bfloat16,fp8,0,0.012537600100040435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,4,128,1,fp8,fp8,0,0.012332800030708312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,8,128,1,bfloat16,bfloat16,0,0.011761599779129028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,8,128,1,bfloat16,fp8,0,0.012225600332021714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,8,128,1,fp8,fp8,0,0.01239679977297783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,1,128,1,bfloat16,bfloat16,0,0.011764799803495407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,1,128,1,bfloat16,fp8,0,0.01265760064125061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,1,128,1,fp8,fp8,0,0.012521600723266602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,2,128,1,bfloat16,bfloat16,0,0.011745599657297134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,2,128,1,bfloat16,fp8,0,0.012353599816560746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,2,128,1,fp8,fp8,0,0.01244639977812767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,4,128,1,bfloat16,bfloat16,0,0.011636800318956374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,4,128,1,bfloat16,fp8,0,0.012415999919176102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,8,4,128,1,fp8,fp8,0,0.012385600060224534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,8,1,128,1,bfloat16,bfloat16,0,0.02133920043706894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,8,1,128,1,bfloat16,fp8,0,0.02299039959907532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,8,1,128,1,fp8,fp8,0,0.022910399734973906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,8,2,128,1,bfloat16,bfloat16,0,0.02824159860610962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,8,2,128,1,bfloat16,fp8,0,0.031379199028015135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,8,2,128,1,fp8,fp8,0,0.03134720027446747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,8,4,128,1,bfloat16,bfloat16,0,0.04288800060749054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,8,4,128,1,bfloat16,fp8,0,0.04896000027656555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,8,4,128,1,fp8,fp8,0,0.04875040054321289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,8,128,1,bfloat16,bfloat16,0,0.04008640050888061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,8,128,1,bfloat16,fp8,0,0.046300798654556274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,8,128,1,fp8,fp8,0,0.04652799963951111
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,1,128,1,bfloat16,bfloat16,0,0.014670400321483612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,1,128,1,bfloat16,fp8,0,0.015332800149917603
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,1,128,1,fp8,fp8,0,0.01539520025253296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,2,128,1,bfloat16,bfloat16,0,0.018236799538135527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,2,128,1,bfloat16,fp8,0,0.019886399805545806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,2,128,1,fp8,fp8,0,0.019912000000476836
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,4,128,1,bfloat16,bfloat16,0,0.02589600086212158
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,4,128,1,bfloat16,fp8,0,0.028825598955154418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,8,4,128,1,fp8,fp8,0,0.02871679961681366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,8,128,1,bfloat16,bfloat16,0,0.02433120012283325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,8,128,1,bfloat16,fp8,0,0.02727360129356384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,8,4,128,1,bfloat16,fp8,0,0.013259199261665345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,8,128,1,fp8,fp8,0,0.027292799949645997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,8,4,128,1,bfloat16,bfloat16,0,0.04678719937801361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,1,128,1,bfloat16,bfloat16,0,0.013606399297714233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,1,128,1,bfloat16,fp8,0,0.014151999354362487
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,1,128,1,fp8,fp8,0,0.014494399726390838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,2,128,1,fp8,fp8,0,0.01414559930562973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,2,128,1,bfloat16,fp8,0,0.014267200231552124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,4,128,1,bfloat16,bfloat16,0,0.01740639954805374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,4,128,1,bfloat16,fp8,0,0.018513600528240203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,4,128,1,fp8,fp8,0,0.01860959976911545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,8,128,1,bfloat16,bfloat16,0,0.016648000478744505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,8,128,1,bfloat16,fp8,0,0.017871999740600587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,8,128,1,fp8,fp8,0,0.017788800597190856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,1,128,1,bfloat16,bfloat16,0,0.012555199861526489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,1,128,1,bfloat16,fp8,0,0.013300800323486328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,1,128,1,fp8,fp8,0,0.01326719969511032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,2,128,1,bfloat16,bfloat16,0,0.012715199589729309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,8,2,128,1,bfloat16,bfloat16,0,0.011816000193357467
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,2,128,1,bfloat16,fp8,0,0.013505600392818451
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,2,128,1,fp8,fp8,0,0.013369600474834441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,4,128,1,bfloat16,bfloat16,0,0.012948800623416901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,4,128,1,bfloat16,fp8,0,0.013468800485134125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,8,128,1,bfloat16,bfloat16,0,0.01228799968957901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,8,128,1,bfloat16,fp8,0,0.013196800649166108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,8,128,1,fp8,fp8,0,0.013076800107955932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,1,128,1,bfloat16,bfloat16,0,0.011905600130558015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,1,128,1,bfloat16,fp8,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,1,128,1,fp8,fp8,0,0.012796799838542938
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,2,128,1,bfloat16,bfloat16,0,0.01204639971256256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,2,128,1,bfloat16,fp8,0,0.012972800433635712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,2,128,1,fp8,fp8,0,0.012950399518013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,4,128,1,bfloat16,bfloat16,0,0.012137600034475327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,4,128,1,bfloat16,fp8,0,0.012964800000190735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,8,4,128,1,fp8,fp8,0,0.013019199669361114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,8,128,1,bfloat16,bfloat16,0,0.012111999839544297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,8,128,1,bfloat16,fp8,0,0.01297920048236847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,8,128,1,fp8,fp8,0,0.01300159990787506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,8,2,128,1,fp8,fp8,0,0.01239359974861145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,1,128,1,bfloat16,bfloat16,0,0.011614400148391723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,1,128,1,bfloat16,fp8,0,0.012507200241088867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,1,128,1,fp8,fp8,0,0.012479999661445617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,2,128,1,bfloat16,bfloat16,0,0.011710400134325028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,2,128,1,bfloat16,fp8,0,0.01268800050020218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,2,128,1,fp8,fp8,0,0.012678399682044983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,4,128,1,bfloat16,fp8,0,0.012567999958992004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,4,128,1,fp8,fp8,0,0.012860800325870513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,8,128,1,bfloat16,bfloat16,0,0.01204639971256256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,8,128,1,bfloat16,fp8,0,0.012425599992275238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,8,128,1,fp8,fp8,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,1,128,1,bfloat16,bfloat16,0,0.011718399822711945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,1,128,1,bfloat16,fp8,0,0.012080000340938568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,1,128,1,fp8,fp8,0,0.01239359974861145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,2,128,1,bfloat16,bfloat16,0,0.01175519973039627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,2,128,1,bfloat16,fp8,0,0.012227199971675873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,2,128,1,fp8,fp8,0,0.012385600060224534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,4,128,1,bfloat16,bfloat16,0,0.011791999638080596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,4,128,1,bfloat16,fp8,0,0.012375999987125397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,8,4,128,1,fp8,fp8,0,0.012559999525547028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,8,128,1,bfloat16,bfloat16,0,0.011800000071525573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,8,128,1,bfloat16,fp8,0,0.012272000312805176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,8,128,1,fp8,fp8,0,0.012572799623012543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,1,128,1,bfloat16,bfloat16,0,0.011540800333023071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,1,128,1,bfloat16,fp8,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,1,128,1,fp8,fp8,0,0.012203200161457062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,2,128,1,bfloat16,bfloat16,0,0.011608000099658965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,2,128,1,bfloat16,fp8,0,0.012238399684429168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,2,128,1,fp8,fp8,0,0.012227199971675873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,4,128,1,bfloat16,bfloat16,0,0.011609599739313126
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,4,128,1,bfloat16,fp8,0,0.012318400293588638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,8,4,128,1,fp8,fp8,0,0.012243200093507767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,8,128,1,bfloat16,bfloat16,0,0.011563199758529662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,8,128,1,bfloat16,fp8,0,0.012193600088357926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,8,128,1,fp8,fp8,0,0.012372799962759019
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,1,128,1,bfloat16,bfloat16,0,0.01143840029835701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,1,128,1,bfloat16,fp8,0,0.012086399644613267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,1,128,1,fp8,fp8,0,0.012272000312805176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,2,128,1,bfloat16,bfloat16,0,0.011574400216341018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,2,128,1,bfloat16,fp8,0,0.012089599668979645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,2,128,1,fp8,fp8,0,0.012278400361537933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,4,128,1,bfloat16,fp8,0,0.012244799733161926
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,4,128,1,fp8,fp8,0,0.012228800356388092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,8,4,128,1,bfloat16,fp8,0,0.013643200695514678
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,8,2,128,1,bfloat16,bfloat16,0,0.013318400084972381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,8,4,128,1,fp8,fp8,0,0.013537600636482239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,8,4,128,1,bfloat16,bfloat16,0,0.012220799922943115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,1,128,1,bfloat16,fp8,0,3.136038398742676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,1,128,1,bfloat16,bfloat16,0,3.5214817047119142
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,8,4,128,1,bfloat16,bfloat16,0,0.011580800265073776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,1,128,1,fp8,fp8,0,3.1594879150390627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,2,128,1,bfloat16,bfloat16,0,3.7276336669921877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,2,128,1,bfloat16,fp8,0,3.137681579589844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,4,128,1,bfloat16,bfloat16,0,1.8047775268554687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,4,128,1,fp8,fp8,0,1.6377504348754883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,4,128,1,bfloat16,fp8,0,1.9057935714721679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,4,2,128,1,fp8,fp8,0,3.1522560119628906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,1,128,1,bfloat16,bfloat16,0,1.735153579711914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,1,128,1,bfloat16,fp8,0,1.6205152511596679
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,4,128,1,bfloat16,bfloat16,0,0.8860639572143555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,1,128,1,fp8,fp8,0,1.6505664825439452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,2,128,1,bfloat16,bfloat16,0,1.7280448913574218
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,4,128,1,fp8,fp8,0,0.8215744018554687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,2,128,1,bfloat16,fp8,0,1.7512496948242187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,4,2,128,1,fp8,fp8,0,1.5848048210144043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,1,128,1,bfloat16,fp8,0,0.7980847835540772
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,1,128,1,bfloat16,bfloat16,0,0.8464127540588379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,1,128,1,fp8,fp8,0,0.8997535705566406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,2,128,1,bfloat16,bfloat16,0,0.869108772277832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,2,128,1,bfloat16,fp8,0,0.839686393737793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,2,128,1,fp8,fp8,0,0.7955071926116943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,4,128,1,bfloat16,bfloat16,0,0.4491568088531494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,4,128,1,bfloat16,fp8,0,0.47048320770263674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,4,128,1,fp8,fp8,0,0.45124001502990724
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,1,128,1,bfloat16,bfloat16,0,0.4297167778015137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,1,128,1,bfloat16,fp8,0,0.4086480140686035
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,1,128,1,fp8,fp8,0,0.39799840450286866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,2,128,1,bfloat16,bfloat16,0,0.43941760063171387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,2,128,1,fp8,fp8,0,0.4275199890136719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,1,128,1,bfloat16,fp8,0,1.8006063461303712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,1,128,1,fp8,fp8,0,1.7991888046264648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,4,4,128,1,bfloat16,fp8,0,0.9320544242858887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,1,128,1,bfloat16,bfloat16,0,2.094396781921387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,2,128,1,bfloat16,fp8,0,1.845395278930664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,2,128,1,bfloat16,bfloat16,0,2.0428672790527345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,4,128,1,bfloat16,bfloat16,0,1.1001903533935546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,4,128,1,fp8,fp8,0,1.0709792137145997
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,1,128,1,bfloat16,fp8,0,1.0469615936279297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,4,2,128,1,fp8,fp8,0,1.8079200744628907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,4,2,128,1,bfloat16,fp8,0,0.44318718910217286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,1,128,1,fp8,fp8,0,0.9887536048889161
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,2,128,1,bfloat16,bfloat16,0,1.0246848106384276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,2,128,1,bfloat16,fp8,0,0.9783519744873047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,4,128,1,bfloat16,fp8,0,0.49857439994812014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,4,128,1,bfloat16,bfloat16,0,0.5788400173187256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,1,128,1,bfloat16,bfloat16,0,0.5151008129119873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,4,128,1,fp8,fp8,0,0.5215695858001709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,1,128,1,bfloat16,fp8,0,0.4765327930450439
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,2,128,1,fp8,fp8,0,0.9286527633666992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,1,128,1,fp8,fp8,0,0.4653471946716309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,4,128,1,bfloat16,bfloat16,0,0.31614720821380615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,2,128,1,bfloat16,bfloat16,0,0.5237279891967773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,2,128,1,bfloat16,fp8,0,0.5062416076660157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,4,128,1,bfloat16,fp8,0,0.2674544095993042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,4,128,1,fp8,fp8,0,0.2636768102645874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,1,128,1,bfloat16,bfloat16,0,0.27302238941192625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,1,128,1,bfloat16,fp8,0,0.2569439888000488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,1,128,1,fp8,fp8,0,0.26567199230194094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,2,128,1,bfloat16,bfloat16,0,0.28147358894348146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,2,128,1,bfloat16,fp8,0,0.25340158939361573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,4,2,128,1,fp8,fp8,0,0.2587536096572876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,4,128,1,bfloat16,fp8,0,0.9729776382446289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,4,1,128,1,bfloat16,bfloat16,0,0.9660703659057617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,1,128,1,bfloat16,fp8,0,1.3127344131469727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,1,128,1,bfloat16,bfloat16,0,1.4132063865661622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,1,128,1,fp8,fp8,0,1.2987055778503418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,2,128,1,bfloat16,fp8,0,1.3013440132141114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,2,128,1,bfloat16,bfloat16,0,1.4276847839355469
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,4,128,1,bfloat16,bfloat16,0,0.7735119819641113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,4,128,1,bfloat16,fp8,0,0.7006671905517579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,4,2,128,1,fp8,fp8,0,1.279248046875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,4,128,1,fp8,fp8,0,0.7617184162139893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,1,128,1,bfloat16,bfloat16,0,0.7081376075744629
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,1,128,1,bfloat16,fp8,0,0.65731201171875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,1,128,1,fp8,fp8,0,0.7457327842712402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,2,128,1,bfloat16,bfloat16,0,0.7204368114471436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,4,128,1,bfloat16,fp8,0,0.3799040079116821
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,4,128,1,fp8,fp8,0,0.365828800201416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,4,128,1,bfloat16,bfloat16,0,0.44681921005249026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,2,128,1,bfloat16,fp8,0,0.6607024192810058
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,4,2,128,1,fp8,fp8,0,0.6514287948608398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,1,128,1,bfloat16,fp8,0,0.3315344095230103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,1,128,1,fp8,fp8,0,0.35166239738464355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,4,2,128,1,fp8,fp8,0,0.4799808025360107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,2,128,1,bfloat16,fp8,0,0.3538352012634277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,2,128,1,bfloat16,bfloat16,0,0.3753727912902832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,4,128,1,bfloat16,fp8,0,0.1883263945579529
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,4,128,1,bfloat16,bfloat16,0,0.24408318996429443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,2,128,1,fp8,fp8,0,0.34013919830322265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,4,128,1,fp8,fp8,0,0.2001471996307373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,1,128,1,bfloat16,bfloat16,0,0.19336479902267456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,1,128,1,bfloat16,fp8,0,0.18794560432434082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,2,128,1,bfloat16,fp8,0,0.18150880336761474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,2,128,1,bfloat16,bfloat16,0,0.20264959335327148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,2,128,1,fp8,fp8,0,0.18445119857788086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,1,128,1,bfloat16,fp8,0,1.6804224014282227
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,1,128,1,fp8,fp8,0,1.7190479278564452
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,2,128,1,bfloat16,fp8,0,1.7216367721557617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,2,128,1,bfloat16,bfloat16,0,1.873396873474121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,4,128,1,bfloat16,fp8,0,0.8859727859497071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,4,128,1,bfloat16,bfloat16,0,0.9748335838317871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,4,128,1,fp8,fp8,0,0.8903535842895508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,4,1,128,1,bfloat16,bfloat16,0,0.3989680051803589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,1,128,1,bfloat16,bfloat16,0,0.993511962890625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,4,1,128,1,fp8,fp8,0,0.18390239477157594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,1,128,1,bfloat16,fp8,0,0.8472736358642579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,1,128,1,fp8,fp8,0,0.913748836517334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,2,128,1,bfloat16,bfloat16,0,0.920809555053711
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,2,128,1,bfloat16,fp8,0,0.85797119140625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,4,2,128,1,fp8,fp8,0,0.9650112152099609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,4,128,1,bfloat16,bfloat16,0,0.490396785736084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,4,128,1,bfloat16,fp8,0,0.4797056198120117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,4,128,1,fp8,fp8,0,0.4945807933807373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,1,128,1,bfloat16,bfloat16,0,0.448964786529541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,1,128,1,bfloat16,fp8,0,0.4868959903717041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,1,128,1,fp8,fp8,0,0.4323296070098877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,2,128,1,bfloat16,bfloat16,0,0.46712641716003417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,2,128,1,bfloat16,fp8,0,0.4581471920013428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,4,2,128,1,fp8,fp8,0,0.43621120452880857
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,4,128,1,bfloat16,bfloat16,0,0.2609695911407471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,4,128,1,bfloat16,fp8,0,0.23279199600219727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,4,128,1,fp8,fp8,0,0.23715200424194335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,1,128,1,bfloat16,bfloat16,0,0.2378175973892212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,1,128,1,bfloat16,fp8,0,0.2187903881072998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,1,128,1,fp8,fp8,0,0.2243119955062866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,2,128,1,bfloat16,bfloat16,0,0.23871359825134278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,2,128,1,bfloat16,fp8,0,0.2236112117767334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,2,128,1,fp8,fp8,0,1.6882400512695312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,4,128,1,bfloat16,fp8,0,0.12110400199890137
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,4,2,128,1,fp8,fp8,0,0.2268928050994873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,4,128,1,fp8,fp8,0,0.11888799667358399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,1,128,1,bfloat16,bfloat16,0,0.11890079975128173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,1,128,1,fp8,fp8,0,0.11800800561904908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,1,128,1,bfloat16,fp8,0,0.11519999504089355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,2,128,1,fp8,fp8,0,0.11640479564666747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,1,128,1,bfloat16,fp8,0,0.9815103530883789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,1,128,1,bfloat16,bfloat16,0,1.0517151832580567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,1,128,1,fp8,fp8,0,0.9894031524658203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,2,128,1,fp8,fp8,0,0.9959039688110352
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,2,128,1,bfloat16,bfloat16,0,1.0851951599121095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,4,128,1,bfloat16,fp8,0,0.5227263927459717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,4,128,1,bfloat16,bfloat16,0,0.5993296146392822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,4,1,128,1,bfloat16,bfloat16,0,1.8266096115112305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,4,128,1,bfloat16,bfloat16,0,0.12799040079116822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,1,128,1,bfloat16,bfloat16,0,0.5253007888793946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,2,128,1,bfloat16,bfloat16,0,0.12143839597702026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,4,2,128,1,bfloat16,fp8,0,0.11491999626159669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,1,128,1,bfloat16,fp8,0,0.5085616111755371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,1,128,1,fp8,fp8,0,0.5044640064239502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,4,128,1,bfloat16,bfloat16,0,0.3178688049316406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,2,128,1,bfloat16,bfloat16,0,0.5463183879852295
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,4,128,1,bfloat16,fp8,0,0.2767215967178345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,2,128,1,fp8,fp8,0,0.5087664127349854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,2,128,1,bfloat16,fp8,0,0.5052495956420898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,4,128,1,fp8,fp8,0,0.27332639694213867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,1,128,1,bfloat16,bfloat16,0,0.2741584062576294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,1,128,1,fp8,fp8,0,0.265881609916687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,1,128,1,bfloat16,fp8,0,0.27620160579681396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,2,128,1,bfloat16,fp8,0,0.26337599754333496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,2,128,1,fp8,fp8,0,0.2628528118133545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,4,128,1,bfloat16,bfloat16,0,0.1727504014968872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,4,128,1,bfloat16,fp8,0,0.14731520414352417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,4,128,1,fp8,fp8,0,0.15299199819564818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,1,128,1,bfloat16,bfloat16,0,0.14745279550552368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,1,128,1,bfloat16,fp8,0,0.13956639766693116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,1,128,1,fp8,fp8,0,0.13952640295028687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,2,128,1,bfloat16,bfloat16,0,0.1557103991508484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,4,128,1,bfloat16,bfloat16,0,0.09456480145454407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,4,128,1,bfloat16,fp8,0,0.08805279731750489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,4,128,1,fp8,fp8,0,0.0895151972770691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,1,128,1,bfloat16,bfloat16,0,0.08884479999542236
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,1,128,1,bfloat16,fp8,0,0.08731359839439393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,1,128,1,fp8,fp8,0,0.08791199922561646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,2,128,1,bfloat16,bfloat16,0,0.0894703984260559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,2,128,1,bfloat16,fp8,0,0.08802080154418945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,4,2,128,1,fp8,fp8,0,0.08811039924621582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,4,2,128,1,bfloat16,fp8,0,1.0160320281982422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,1,128,1,bfloat16,fp8,0,0.9316304206848145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,1,128,1,fp8,fp8,0,0.938599967956543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,2,128,1,bfloat16,bfloat16,0,1.0300432205200196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,4,4,128,1,fp8,fp8,0,0.553433609008789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,2,128,1,bfloat16,fp8,0,0.9384480476379394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,4,128,1,bfloat16,bfloat16,0,0.592083215713501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,2,128,1,fp8,fp8,0,0.946190357208252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,4,128,1,bfloat16,fp8,0,0.5122672080993652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,4,128,1,fp8,fp8,0,0.506115198135376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,4,2,128,1,bfloat16,bfloat16,0,0.28193440437316897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,1,128,1,bfloat16,fp8,0,0.4777679920196533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,1,128,1,fp8,fp8,0,0.46820797920227053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,2,128,1,bfloat16,fp8,0,0.14261599779129028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,4,2,128,1,fp8,fp8,0,0.13931360244750976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,2,128,1,bfloat16,bfloat16,0,0.5173888206481934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,4,128,1,bfloat16,bfloat16,0,0.28894240856170655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,2,128,1,bfloat16,fp8,0,0.481217622756958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,2,128,1,fp8,fp8,0,0.4817296028137207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,4,128,1,bfloat16,fp8,0,0.2628047943115234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,1,128,1,bfloat16,bfloat16,0,0.25232160091400146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,4,128,1,fp8,fp8,0,0.26007039546966554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,1,128,1,fp8,fp8,0,0.2452608108520508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,2,128,1,bfloat16,bfloat16,0,0.26776320934295655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,4,128,1,bfloat16,bfloat16,0,0.1507856011390686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,2,128,1,bfloat16,fp8,0,0.24895999431610108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,2,128,1,fp8,fp8,0,0.24560799598693847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,4,128,1,bfloat16,fp8,0,0.13775360584259033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,4,128,1,fp8,fp8,0,0.13685920238494872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,1,128,1,bfloat16,bfloat16,0,0.13214240074157715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,1,128,1,bfloat16,fp8,0,0.1265023946762085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,1,128,1,fp8,fp8,0,0.12446719408035278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,2,128,1,bfloat16,bfloat16,0,0.13888479471206666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,2,128,1,bfloat16,fp8,0,0.1277008056640625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,4,128,1,bfloat16,bfloat16,0,0.07799040079116822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,4,128,1,bfloat16,fp8,0,0.0700111985206604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,4,128,1,fp8,fp8,0,0.06969919800758362
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,1,128,1,bfloat16,bfloat16,0,0.06735680103302003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,1,128,1,bfloat16,fp8,0,0.06820480227470398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,1,128,1,fp8,fp8,0,0.06725760102272034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,2,128,1,bfloat16,bfloat16,0,0.0690768003463745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,4,1,128,1,bfloat16,bfloat16,0,0.9635392189025879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,2,128,1,bfloat16,fp8,0,0.06826239824295044
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,4,2,128,1,fp8,fp8,0,0.06744959950447083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,4,128,1,bfloat16,bfloat16,0,0.06391199827194213
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,4,128,1,fp8,fp8,0,0.06254879832267761
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,4,128,1,bfloat16,fp8,0,0.06239680051803589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,1,128,1,bfloat16,fp8,0,0.06120960116386413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,1,128,1,fp8,fp8,0,0.06180319786071777
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,2,128,1,bfloat16,bfloat16,0,0.06316800117492676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,2,128,1,bfloat16,fp8,0,0.061084800958633424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,2,128,1,fp8,fp8,0,0.061484801769256595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,1,128,1,bfloat16,bfloat16,0,0.6356063842773437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,1,128,1,bfloat16,fp8,0,0.6083983898162841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,1,128,1,fp8,fp8,0,0.6115439891815185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,4,1,128,1,bfloat16,bfloat16,0,0.5241375923156738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,2,128,1,bfloat16,bfloat16,0,0.6684624195098877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,2,128,1,fp8,fp8,0,0.6198128223419189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,4,128,1,bfloat16,bfloat16,0,0.38287041187286375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,4,1,128,1,bfloat16,fp8,0,0.2410975933074951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,4,128,1,bfloat16,fp8,0,0.3446912050247192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,1,128,1,bfloat16,bfloat16,0,0.3309216022491455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,4,128,1,fp8,fp8,0,0.3407599925994873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,1,128,1,bfloat16,fp8,0,0.31688799858093264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,1,128,1,fp8,fp8,0,0.31831040382385256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,2,128,1,bfloat16,bfloat16,0,0.3476495981216431
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,2,128,1,bfloat16,fp8,0,0.31918399333953856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,4,2,128,1,fp8,fp8,0,0.12853120565414428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,4,128,1,bfloat16,bfloat16,0,0.20240480899810792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,4,128,1,bfloat16,fp8,0,0.1809216022491455
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,4,128,1,fp8,fp8,0,0.17983839511871338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,4,2,128,1,fp8,fp8,0,0.32115840911865234
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,4,1,128,1,bfloat16,bfloat16,0,0.062428802251815796
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,1,128,1,bfloat16,fp8,0,0.16942720413208007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,1,128,1,fp8,fp8,0,0.1693295955657959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,2,128,1,bfloat16,bfloat16,0,0.18327200412750244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,2,128,1,bfloat16,fp8,0,0.1708624005317688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,2,128,1,fp8,fp8,0,0.1688704013824463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,4,128,1,bfloat16,bfloat16,0,0.10831199884414673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,4,128,1,fp8,fp8,0,0.0989247977733612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,4,128,1,bfloat16,fp8,0,0.09933279752731324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,1,128,1,bfloat16,fp8,0,0.0924880027770996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,1,128,1,fp8,fp8,0,0.09113280177116394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,2,128,1,bfloat16,fp8,0,0.09289439916610717
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,2,128,1,fp8,fp8,0,0.09349759817123413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,4,128,1,bfloat16,bfloat16,0,0.061947202682495116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,4,128,1,bfloat16,fp8,0,0.055163198709487916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,4,128,1,fp8,fp8,0,0.05577759742736817
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,1,128,1,bfloat16,bfloat16,0,0.05517280101776123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,1,128,1,bfloat16,fp8,0,0.055137598514556886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,1,128,1,fp8,fp8,0,0.054816001653671266
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,2,128,1,bfloat16,bfloat16,0,0.055195200443267825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,2,128,1,bfloat16,fp8,0,0.05536159873008728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,4,2,128,1,fp8,fp8,0,0.05545920133590698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,4,128,1,bfloat16,bfloat16,0,0.053092801570892335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,4,128,1,bfloat16,fp8,0,0.05136160254478454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,4,128,1,fp8,fp8,0,0.05148000121116638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,1,128,1,bfloat16,bfloat16,0,0.04216319918632507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,1,128,1,bfloat16,fp8,0,0.04119040071964264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,1,128,1,fp8,fp8,0,0.04157919883728027
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,2,128,1,bfloat16,bfloat16,0,0.048497599363327024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,2,128,1,bfloat16,fp8,0,0.04881280064582825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,4,2,128,1,fp8,fp8,0,0.04819679856300354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,4,2,128,1,bfloat16,fp8,0,0.6159759998321533
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,1,128,1,bfloat16,bfloat16,0,0.6313327789306641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,1,128,1,bfloat16,fp8,0,0.6173344135284424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,1,128,1,fp8,fp8,0,0.612001609802246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,2,128,1,bfloat16,bfloat16,0,0.6833360195159912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,2,128,1,bfloat16,fp8,0,0.6246575832366943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,4,128,1,bfloat16,fp8,0,0.35386559963226316
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,4,128,1,fp8,fp8,0,0.35447680950164795
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,4,2,128,1,fp8,fp8,0,0.6240655899047851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,1,128,1,bfloat16,bfloat16,0,0.3291055917739868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,1,128,1,bfloat16,fp8,0,0.3176944017410278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,1,128,1,fp8,fp8,0,0.3180543899536133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,2,128,1,bfloat16,bfloat16,0,0.35106720924377444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,2,128,1,bfloat16,fp8,0,0.32472639083862304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,4,128,1,bfloat16,bfloat16,0,0.21268000602722167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,2,128,1,fp8,fp8,0,0.3237648010253906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,4,128,1,bfloat16,fp8,0,0.18856480121612548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,4,128,1,fp8,fp8,0,0.1887328028678894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,1,128,1,bfloat16,bfloat16,0,0.17843040227890014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,1,128,1,bfloat16,bfloat16,0,0.09643200039863586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,1,128,1,bfloat16,fp8,0,0.16980639696121216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,1,128,1,fp8,fp8,0,0.17029759883880616
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,2,128,1,bfloat16,bfloat16,0,0.18861440420150757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,2,128,1,bfloat16,fp8,0,0.17252320051193237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,4,128,1,bfloat16,fp8,0,0.10693440437316895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,4,2,128,1,fp8,fp8,0,0.1745151996612549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,4,128,1,fp8,fp8,0,0.1062127947807312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,1,128,1,bfloat16,bfloat16,0,0.09994239807128906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,1,128,1,bfloat16,fp8,0,0.09515519738197327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,1,128,1,fp8,fp8,0,0.09532319903373718
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,2,128,1,bfloat16,bfloat16,0,0.10560319423675538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,2,128,1,bfloat16,fp8,0,0.09733440279960633
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,2,128,1,fp8,fp8,0,0.09746080040931701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,4,128,1,bfloat16,bfloat16,0,0.0692031979560852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,4,128,1,bfloat16,fp8,0,0.060099202394485476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,4,128,1,fp8,fp8,0,0.061153602600097653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,1,128,1,bfloat16,bfloat16,0,0.05861120223999024
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,1,128,1,bfloat16,fp8,0,0.058582401275634764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,1,128,1,fp8,fp8,0,0.058764797449111936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,2,128,1,bfloat16,bfloat16,0,0.06099519729614258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,2,128,1,fp8,fp8,0,0.05882560014724732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,4,128,1,bfloat16,bfloat16,0,0.04196479916572571
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,4,128,1,bfloat16,fp8,0,0.040761598944664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,4,128,1,fp8,fp8,0,0.04040960073471069
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,1,128,1,bfloat16,bfloat16,0,0.040463998913764954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,1,128,1,bfloat16,fp8,0,0.0402319997549057
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,1,128,1,fp8,fp8,0,0.040238401293754576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,2,128,1,bfloat16,bfloat16,0,0.04062559902667999
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,2,128,1,bfloat16,fp8,0,0.040755200386047366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,4,2,128,1,fp8,fp8,0,0.040166398882865904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,4,128,1,bfloat16,bfloat16,0,0.032924801111221313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,4,128,1,bfloat16,fp8,0,0.032662400603294374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,4,128,1,fp8,fp8,0,0.0328031986951828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,1,128,1,bfloat16,fp8,0,0.032262399792671204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,4,4,128,1,bfloat16,bfloat16,0,0.3996592044830322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,1,128,1,fp8,fp8,0,0.03237279951572418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,2,128,1,bfloat16,bfloat16,0,0.03215999901294708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,2,128,1,bfloat16,fp8,0,0.03265120089054108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,2,128,1,fp8,fp8,0,0.03245759904384613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,4,1,128,1,bfloat16,bfloat16,0,0.1753999948501587
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,1,128,1,bfloat16,bfloat16,0,0.4066287994384766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,1,128,1,bfloat16,fp8,0,0.3967375993728638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,1,128,1,fp8,fp8,0,0.3959791898727417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,4,2,128,1,bfloat16,bfloat16,0,0.10109440088272095
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,4,4,128,1,bfloat16,bfloat16,0,0.11631200313568116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,2,128,1,bfloat16,bfloat16,0,0.4363743782043457
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,2,128,1,bfloat16,fp8,0,0.40635042190551757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,4,128,1,bfloat16,bfloat16,0,0.265231990814209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,4,2,128,1,fp8,fp8,0,0.40559840202331543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,4,128,1,bfloat16,fp8,0,0.23570079803466798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,1,128,1,bfloat16,bfloat16,0,0.2161087989807129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,1,128,1,bfloat16,fp8,0,0.2077455997467041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,1,128,1,fp8,fp8,0,0.20876479148864746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,2,128,1,bfloat16,bfloat16,0,0.23265600204467773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,4,2,128,1,bfloat16,fp8,0,0.05860000252723694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,2,128,1,bfloat16,fp8,0,0.21396479606628419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,2,128,1,fp8,fp8,0,0.2132688045501709
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,4,128,1,bfloat16,bfloat16,0,0.1435744047164917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,4,128,1,bfloat16,fp8,0,0.1268064022064209
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,4,128,1,fp8,fp8,0,0.128220796585083
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,1,128,1,bfloat16,bfloat16,0,0.11847519874572754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,1,128,1,bfloat16,fp8,0,0.11329439878463746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,2,128,1,bfloat16,bfloat16,0,0.12654240131378175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,1,128,1,fp8,fp8,0,0.11466399431228638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,2,128,1,bfloat16,fp8,0,0.11771039962768555
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,4,2,128,1,fp8,fp8,0,0.11738719940185546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,4,128,1,bfloat16,bfloat16,0,0.0806335985660553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,4,128,1,bfloat16,fp8,0,0.07483999729156494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,4,128,1,fp8,fp8,0,0.07462559938430786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,1,128,1,bfloat16,bfloat16,0,0.06854239702224732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,1,128,1,bfloat16,fp8,0,0.06584640145301819
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,1,128,1,fp8,fp8,0,0.06578400135040283
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,2,128,1,bfloat16,bfloat16,0,0.0735360026359558
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,2,128,1,bfloat16,fp8,0,0.06640160083770752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,4,1,128,1,bfloat16,bfloat16,0,0.03187839984893799
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,4,2,128,1,fp8,fp8,0,0.06746559739112853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,4,128,1,bfloat16,bfloat16,0,0.04907999932765961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,4,128,1,bfloat16,fp8,0,0.04305280148983002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,4,128,1,fp8,fp8,0,0.043403199315071105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,1,128,1,bfloat16,bfloat16,0,0.041403201222419736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,1,128,1,bfloat16,fp8,0,0.04223200082778931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,2,128,1,bfloat16,bfloat16,0,0.042182400822639465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,1,128,1,fp8,fp8,0,0.04232319891452789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,2,128,1,bfloat16,fp8,0,0.04235680103302002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,4,2,128,1,fp8,fp8,0,0.04279040098190308
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,4,128,1,bfloat16,fp8,0,0.040140798687934874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,4,128,1,fp8,fp8,0,0.03993920087814331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,1,128,1,bfloat16,bfloat16,0,0.036985599994659425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,1,128,1,bfloat16,fp8,0,0.03736160099506378
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,1,128,1,fp8,fp8,0,0.03769600093364715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,2,128,1,bfloat16,bfloat16,0,0.03894400000572205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,2,128,1,bfloat16,fp8,0,0.03951840102672577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,4,128,1,bfloat16,bfloat16,0,0.026419198513031004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,4,128,1,bfloat16,fp8,0,0.02651360034942627
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,4,128,1,fp8,fp8,0,0.02648639976978302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,1,128,1,bfloat16,bfloat16,0,0.024644799530506134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,1,128,1,bfloat16,fp8,0,0.026054400205612182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,1,128,1,fp8,fp8,0,0.025964799523353576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,2,128,1,bfloat16,bfloat16,0,0.025054401159286498
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,2,128,1,bfloat16,fp8,0,0.026128000020980834
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,4,2,128,1,fp8,fp8,0,0.0261135995388031
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,1,128,1,bfloat16,fp8,0,0.4302879810333252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,1,128,1,bfloat16,bfloat16,0,0.4360640048980713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,1,128,1,fp8,fp8,0,0.4296127796173096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,2,128,1,bfloat16,bfloat16,0,0.4779263973236084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,2,128,1,bfloat16,fp8,0,0.4429935932159424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,4,4,128,1,fp8,fp8,0,0.23348801136016845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,4,128,1,bfloat16,fp8,0,0.26196320056915284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,4,128,1,fp8,fp8,0,0.2612384080886841
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,1,128,1,bfloat16,bfloat16,0,0.22811360359191896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,1,128,1,bfloat16,fp8,0,0.22510080337524413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,1,128,1,fp8,fp8,0,0.22361280918121337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,2,128,1,bfloat16,bfloat16,0,0.25028960704803466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,2,128,1,bfloat16,fp8,0,0.23117599487304688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,4,128,1,bfloat16,bfloat16,0,0.15804799795150756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,2,128,1,fp8,fp8,0,0.23070878982543946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,4,128,1,bfloat16,fp8,0,0.1400223970413208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,4,128,1,fp8,fp8,0,0.14074239730834961
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,1,128,1,bfloat16,bfloat16,0,0.12270400524139405
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,1,128,1,bfloat16,fp8,0,0.12156319618225098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,1,128,1,fp8,fp8,0,0.12202880382537842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,2,128,1,bfloat16,bfloat16,0,0.13385119438171386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,4,128,1,bfloat16,bfloat16,0,0.041387200355529785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,2,128,1,bfloat16,fp8,0,0.12437920570373535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,4,128,1,bfloat16,bfloat16,0,0.08684319853782654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,4,2,128,1,fp8,fp8,0,0.12583520412445068
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,4,128,1,bfloat16,fp8,0,0.07968800067901612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,4,2,128,1,fp8,fp8,0,0.03984479904174805
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,4,128,1,fp8,fp8,0,0.08050400018692017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,1,128,1,bfloat16,bfloat16,0,0.07015200257301331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,1,128,1,bfloat16,fp8,0,0.06803359985351562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,1,128,1,fp8,fp8,0,0.06802080273628235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,2,128,1,bfloat16,fp8,0,0.07123519778251648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,2,128,1,fp8,fp8,0,0.07019519805908203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,4,128,1,bfloat16,bfloat16,0,0.053572797775268556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,4,128,1,fp8,fp8,0,0.044670400023460385
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,1,128,1,bfloat16,bfloat16,0,0.04256959855556488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,1,128,1,bfloat16,fp8,0,0.04315359890460968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,1,128,1,fp8,fp8,0,0.043137601017951964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,2,128,1,bfloat16,bfloat16,0,0.04433279931545257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,2,128,1,bfloat16,fp8,0,0.04318400025367737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,2,128,1,fp8,fp8,0,0.0432096004486084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,4,128,1,bfloat16,bfloat16,0,0.02945919930934906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,4,128,1,bfloat16,fp8,0,0.029100799560546876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,4,128,1,fp8,fp8,0,0.028856000304222106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,1,128,1,bfloat16,bfloat16,0,0.027529600262641906
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,1,128,1,bfloat16,fp8,0,0.028519999980926514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,1,128,1,fp8,fp8,0,0.028646400570869444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,2,128,1,bfloat16,bfloat16,0,0.027878400683403016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,2,128,1,bfloat16,fp8,0,0.028420799970626832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,4,2,128,1,fp8,fp8,0,0.02810879945755005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,4,128,1,bfloat16,bfloat16,0,0.026366400718688964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,4,128,1,bfloat16,fp8,0,0.02739199995994568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,4,128,1,fp8,fp8,0,0.027188798785209654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,1,128,1,bfloat16,bfloat16,0,0.02555519938468933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,1,128,1,bfloat16,fp8,0,0.027024000883102417
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,1,128,1,fp8,fp8,0,0.02728799879550934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,2,128,1,bfloat16,bfloat16,0,0.025723201036453248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,2,128,1,bfloat16,fp8,0,0.027001601457595826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,4,2,128,1,fp8,fp8,0,0.027065598964691163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,4,128,1,bfloat16,bfloat16,0,0.021480000019073485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,4,128,1,bfloat16,fp8,0,0.021206399798393248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,4,128,1,fp8,fp8,0,0.021187199652194975
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,1,128,1,bfloat16,bfloat16,0,0.021115200221538545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,1,128,1,bfloat16,fp8,0,0.020755200088024138
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,4,2,128,1,fp8,fp8,0,0.44300642013549807
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,1,128,1,fp8,fp8,0,0.0208064004778862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,2,128,1,bfloat16,bfloat16,0,0.02123199999332428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,2,128,1,bfloat16,fp8,0,0.020895999670028687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,4,2,128,1,fp8,fp8,0,0.02067520022392273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,1,128,1,bfloat16,bfloat16,0,0.28435521125793456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,1,128,1,fp8,fp8,0,0.2902031898498535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,2,128,1,bfloat16,bfloat16,0,0.329585599899292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,2,128,1,bfloat16,fp8,0,0.3037760019302368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,2,128,1,fp8,fp8,0,0.3042992115020752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,4,128,1,bfloat16,bfloat16,0,0.22089920043945313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,4,128,1,bfloat16,fp8,0,0.19227360486984252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,4,128,1,fp8,fp8,0,0.19148160219192506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,1,128,1,bfloat16,bfloat16,0,0.14973280429840088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,1,128,1,bfloat16,fp8,0,0.15201280117034913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,1,128,1,fp8,fp8,0,0.15343999862670898
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,2,128,1,bfloat16,bfloat16,0,0.1731503963470459
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,2,128,1,bfloat16,fp8,0,0.16001759767532348
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,4,2,128,1,fp8,fp8,0,0.15862879753112794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,4,128,1,bfloat16,bfloat16,0,0.118721604347229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,4,128,1,bfloat16,fp8,0,0.10306240320205688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,4,128,1,fp8,fp8,0,0.10356639623641968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,1,128,1,bfloat16,bfloat16,0,0.0820688009262085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,1,128,1,bfloat16,fp8,0,0.08467680215835571
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,1,128,1,fp8,fp8,0,0.08496159911155701
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,2,128,1,bfloat16,bfloat16,0,0.09322239756584168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,2,128,1,bfloat16,fp8,0,0.0871392011642456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,4,2,128,1,fp8,fp8,0,0.08856160044670106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,4,2,128,1,bfloat16,bfloat16,0,0.07544320225715637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,4,128,1,bfloat16,bfloat16,0,0.0671999990940094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,4,128,1,bfloat16,fp8,0,0.05899199843406677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,4,128,1,fp8,fp8,0,0.05924479961395264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,4,4,128,1,bfloat16,fp8,0,0.045070400834083556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,1,128,1,bfloat16,bfloat16,0,0.04881759881973267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,1,128,1,bfloat16,fp8,0,0.0471807986497879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,1,128,1,fp8,fp8,0,0.04770880043506622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,2,128,1,bfloat16,fp8,0,0.050323200225830075
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,2,128,1,bfloat16,bfloat16,0,0.05387200117111206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,4,2,128,1,fp8,fp8,0,0.050190401077270505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,4,128,1,bfloat16,fp8,0,0.03261120021343231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,4,128,1,fp8,fp8,0,0.03253760039806366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,1,128,1,bfloat16,bfloat16,0,0.02828960120677948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,1,128,1,bfloat16,fp8,0,0.02933279871940613
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,1,128,1,fp8,fp8,0,0.02953920066356659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,2,128,1,bfloat16,bfloat16,0,0.029900801181793214
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,2,128,1,bfloat16,fp8,0,0.030142399668693542
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,2,128,1,fp8,fp8,0,0.02968960106372833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,4,128,1,bfloat16,fp8,0,0.020313599705696107
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,4,128,1,fp8,fp8,0,0.020558400452136992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,1,128,1,bfloat16,bfloat16,0,0.01841920018196106
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,1,128,1,bfloat16,fp8,0,0.019817599654197694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,1,128,1,fp8,fp8,0,0.02001439929008484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,2,128,1,bfloat16,bfloat16,0,0.018807999789714813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,2,128,1,bfloat16,fp8,0,0.019871999323368073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,2,128,1,fp8,fp8,0,0.020044800639152528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,4,128,1,bfloat16,bfloat16,0,0.01839040070772171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,4,128,1,bfloat16,fp8,0,0.019516800343990327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,4,128,1,fp8,fp8,0,0.019756799936294554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,1,128,1,bfloat16,bfloat16,0,0.017371200025081635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,1,128,1,bfloat16,fp8,0,0.018889600038528444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,1,128,1,fp8,fp8,0,0.01930239945650101
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,2,128,1,bfloat16,fp8,0,0.01914079934358597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,2,128,1,fp8,fp8,0,0.019518400728702544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,4,128,1,bfloat16,bfloat16,0,0.018004800379276275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,4,128,1,bfloat16,fp8,0,0.019569599628448488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,4,128,1,fp8,fp8,0,0.01926880031824112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,1,128,1,bfloat16,bfloat16,0,0.01746399998664856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,1,128,1,bfloat16,fp8,0,0.018966400623321535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,1,128,1,fp8,fp8,0,0.019014400243759156
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,2,128,1,bfloat16,bfloat16,0,0.01748960018157959
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,2,128,1,bfloat16,fp8,0,0.0191103994846344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,4,2,128,1,fp8,fp8,0,0.018907199800014495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,4,128,1,bfloat16,bfloat16,0,0.01716800034046173
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,4,128,1,bfloat16,fp8,0,0.018803200125694274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,4,128,1,fp8,fp8,0,0.019075199961662292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,1,128,1,bfloat16,bfloat16,0,0.016771200299263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,1,128,1,bfloat16,fp8,0,0.018799999356269838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,1,128,1,fp8,fp8,0,0.018475200235843658
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,4,1,128,1,bfloat16,fp8,0,0.29035201072692873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,2,128,1,bfloat16,bfloat16,0,0.017472000420093538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,2,128,1,bfloat16,fp8,0,0.018603199720382692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,4,2,128,1,fp8,fp8,0,0.018777599930763243
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,1,128,1,bfloat16,bfloat16,0,0.12943520545959472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,1,128,1,bfloat16,fp8,0,0.13091520071029664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,1,128,1,fp8,fp8,0,0.13161120414733887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,2,128,1,bfloat16,bfloat16,0,0.15472960472106934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,2,128,1,bfloat16,fp8,0,0.13853919506072998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,4,2,128,1,fp8,fp8,0,0.13903360366821288
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,4,128,1,bfloat16,bfloat16,0,0.11218080520629883
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,4,128,1,fp8,fp8,0,0.09013440012931824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,1,128,1,bfloat16,bfloat16,0,0.07278239727020264
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,1,128,1,bfloat16,fp8,0,0.07244480252265931
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,1,128,1,fp8,fp8,0,0.07310079932212829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,2,128,1,bfloat16,bfloat16,0,0.08518400192260742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,2,128,1,bfloat16,fp8,0,0.07745919823646545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,2,128,1,fp8,fp8,0,0.07674559950828552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,4,128,1,bfloat16,bfloat16,0,0.06325920224189759
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,4,128,1,bfloat16,fp8,0,0.05109279751777649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,4,128,1,fp8,fp8,0,0.051070398092269896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,1,128,1,bfloat16,bfloat16,0,0.04232319891452789
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,1,128,1,bfloat16,fp8,0,0.0404911994934082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,1,128,1,fp8,fp8,0,0.04065600037574768
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,4,4,128,1,bfloat16,bfloat16,0,0.03903360068798065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,2,128,1,bfloat16,bfloat16,0,0.04832639992237091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,2,128,1,bfloat16,fp8,0,0.04389919936656952
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,4,2,128,1,fp8,fp8,0,0.04394400119781494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,4,128,1,bfloat16,bfloat16,0,0.03467040061950684
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,4,128,1,bfloat16,fp8,0,0.026636800169944762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,4,128,1,fp8,fp8,0,0.026769599318504332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,1,128,1,bfloat16,bfloat16,0,0.023259200155735016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,1,128,1,bfloat16,fp8,0,0.025113600492477416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,1,128,1,fp8,fp8,0,0.02510719895362854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,2,128,1,bfloat16,bfloat16,0,0.025041601061820982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,2,128,1,bfloat16,fp8,0,0.025255998969078063
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,4,2,128,1,fp8,fp8,0,0.025248000025749208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,4,128,1,bfloat16,bfloat16,0,0.017419199645519256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,4,128,1,bfloat16,fp8,0,0.01711200028657913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,4,128,1,fp8,fp8,0,0.01687999963760376
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,1,128,1,bfloat16,bfloat16,0,0.015547199547290802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,1,128,1,bfloat16,fp8,0,0.016288000345230102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,1,128,1,fp8,fp8,0,0.016251200437545778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,4,2,128,1,bfloat16,bfloat16,0,0.017764799296855927
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,2,128,1,bfloat16,bfloat16,0,0.015883199870586395
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,2,128,1,bfloat16,fp8,0,0.016467200219631196
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,4,2,128,1,fp8,fp8,0,0.01648640036582947
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,4,128,1,bfloat16,bfloat16,0,0.015803200006484986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,4,128,1,bfloat16,fp8,0,0.016235199570655823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,1,128,1,bfloat16,bfloat16,0,0.014659200608730317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,1,128,1,bfloat16,fp8,0,0.015388800203800202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,1,128,1,fp8,fp8,0,0.01563519984483719
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,2,128,1,bfloat16,bfloat16,0,0.015150399506092071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,2,128,1,bfloat16,fp8,0,0.01568640023469925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,4,128,1,bfloat16,bfloat16,0,0.014860799908638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,4,4,128,1,bfloat16,bfloat16,0,0.2970959901809692
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,4,128,1,bfloat16,fp8,0,0.015667200088500977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,4,128,1,fp8,fp8,0,0.01566559970378876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,1,128,1,bfloat16,fp8,0,0.01512639969587326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,1,128,1,fp8,fp8,0,0.015156799554824829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,1,128,1,bfloat16,bfloat16,0,0.014299200475215912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,2,128,1,bfloat16,bfloat16,0,0.01465280055999756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,2,128,1,bfloat16,fp8,0,0.015291200578212738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,4,2,128,1,fp8,fp8,0,0.015380799770355225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,4,128,1,bfloat16,bfloat16,0,0.014276799559593201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,4,128,1,bfloat16,fp8,0,0.015123200416564942
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,4,128,1,fp8,fp8,0,0.015217599272727967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,1,128,1,bfloat16,bfloat16,0,0.014001600444316864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,1,128,1,bfloat16,fp8,0,0.01485760062932968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,1,128,1,fp8,fp8,0,0.015012800693511963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,2,128,1,bfloat16,bfloat16,0,0.014347200095653535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,2,128,1,bfloat16,fp8,0,0.015017600357532501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,4,2,128,1,fp8,fp8,0,0.015108799934387207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,4,128,1,bfloat16,bfloat16,0,0.013929599523544311
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,4,128,1,bfloat16,fp8,0,0.01491519957780838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,4,128,1,fp8,fp8,0,0.014856000244617463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,1,128,1,bfloat16,bfloat16,0,0.013870400190353394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,1,128,1,bfloat16,fp8,0,0.014753599464893342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,1,128,1,fp8,fp8,0,0.01481119990348816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,2,128,1,bfloat16,bfloat16,0,0.013831999897956849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,2,128,1,bfloat16,fp8,0,0.015006400644779205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,4,2,128,1,fp8,fp8,0,0.014841599762439728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,4,4,128,1,bfloat16,fp8,0,0.08964959979057312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,1,128,1,bfloat16,bfloat16,0,0.07157440185546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,1,128,1,bfloat16,fp8,0,0.07184960246086121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,1,128,1,fp8,fp8,0,0.07050399780273438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,2,128,1,bfloat16,bfloat16,0,0.08577280044555664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,2,128,1,bfloat16,fp8,0,0.07539680004119872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,4,2,128,1,fp8,fp8,0,0.07396640181541443
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,4,128,1,bfloat16,bfloat16,0,0.06291840076446534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,4,128,1,bfloat16,fp8,0,0.047972801327705386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,4,128,1,fp8,fp8,0,0.04752320051193237
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,1,128,1,bfloat16,bfloat16,0,0.04068160057067871
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,1,128,1,bfloat16,fp8,0,0.0372624009847641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,2,128,1,bfloat16,bfloat16,0,0.046777600049972536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,2,128,1,bfloat16,fp8,0,0.04085760116577149
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,2,128,1,fp8,fp8,0,0.040468800067901614
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,4,128,1,bfloat16,bfloat16,0,0.03274079859256744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,4,128,1,bfloat16,fp8,0,0.02250880002975464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,4,128,1,fp8,fp8,0,0.022908799350261688
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,1,128,1,bfloat16,bfloat16,0,0.021209600567817687
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,1,128,1,bfloat16,fp8,0,0.02210559993982315
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,1,128,1,fp8,fp8,0,0.02232639938592911
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,2,128,1,bfloat16,bfloat16,0,0.022416000068187714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,2,128,1,bfloat16,fp8,0,0.022334399819374084
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,4,2,128,1,fp8,fp8,0,0.022329600155353548
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,4,128,1,bfloat16,bfloat16,0,0.01634719967842102
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,4,128,1,bfloat16,fp8,0,0.015825599431991577
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,4,128,1,fp8,fp8,0,0.015561600029468537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,1,128,1,bfloat16,bfloat16,0,0.014703999459743499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,1,128,1,fp8,fp8,0,0.015203200280666351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,2,128,1,bfloat16,bfloat16,0,0.015043200552463531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,2,128,1,bfloat16,fp8,0,0.015252800285816192
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,2,128,1,fp8,fp8,0,0.01530880033969879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,4,128,1,bfloat16,bfloat16,0,0.01422239989042282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,4,128,1,bfloat16,fp8,0,0.014076800644397735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,4,128,1,fp8,fp8,0,0.0139055997133255
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,1,128,1,bfloat16,bfloat16,0,0.01302880048751831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,1,128,1,bfloat16,fp8,0,0.013692800700664521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,1,128,1,fp8,fp8,0,0.013617600500583648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,2,128,1,bfloat16,bfloat16,0,0.013630400598049163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,2,128,1,bfloat16,fp8,0,0.013755199313163758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,4,128,1,fp8,fp8,0,0.01600960046052933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,4,2,128,1,fp8,fp8,0,0.013792000710964203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,4,128,1,bfloat16,bfloat16,0,0.013504000008106231
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,4,128,1,bfloat16,fp8,0,0.013784000277519226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,4,128,1,fp8,fp8,0,0.013849599659442902
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,1,128,1,bfloat16,fp8,0,0.013580800592899322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,4,2,128,1,fp8,fp8,0,0.01597920060157776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,1,128,1,fp8,fp8,0,0.013628800213336945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,2,128,1,bfloat16,bfloat16,0,0.013068799674510957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,2,128,1,bfloat16,fp8,0,0.01371999979019165
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,4,128,1,bfloat16,bfloat16,0,0.012913599610328674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,4,128,1,bfloat16,fp8,0,0.013350400328636169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,4,128,1,fp8,fp8,0,0.013073599338531494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,1,128,1,bfloat16,bfloat16,0,0.012654399871826172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,1,128,1,bfloat16,fp8,0,0.013102400302886962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,1,128,1,fp8,fp8,0,0.013099199533462525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,2,128,1,bfloat16,bfloat16,0,0.012353599816560746
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,2,128,1,bfloat16,fp8,0,0.013091200590133667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,4,2,128,1,fp8,fp8,0,0.012801599502563477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,4,128,1,bfloat16,bfloat16,0,0.012438400089740754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,4,128,1,bfloat16,fp8,0,0.012702399492263794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,4,128,1,fp8,fp8,0,0.012865599989891053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,1,128,1,bfloat16,bfloat16,0,0.01202239990234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,1,128,1,bfloat16,fp8,0,0.012865599989891053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,1,128,1,fp8,fp8,0,0.012772800028324127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,2,128,1,bfloat16,bfloat16,0,0.012388800084590913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,2,128,1,bfloat16,fp8,0,0.012740799784660339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,4,2,128,1,fp8,fp8,0,0.01297439932823181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,4,128,1,bfloat16,fp8,0,0.012863999605178833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,4,128,1,fp8,fp8,0,0.012873600423336028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,1,128,1,bfloat16,bfloat16,0,0.01220960021018982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,1,128,1,bfloat16,fp8,0,0.012828800082206725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,1,128,1,fp8,fp8,0,0.012824000418186187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,2,128,1,bfloat16,bfloat16,0,0.012174399942159653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,2,128,1,bfloat16,fp8,0,0.01282079964876175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,2,128,1,fp8,fp8,0,0.01284639984369278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,1,128,1,bfloat16,bfloat16,0,0.04160319864749908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,4,1,128,1,fp8,fp8,0,0.03747200071811676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,1,128,1,bfloat16,fp8,0,0.03950720131397247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,1,128,1,fp8,fp8,0,0.03893440067768097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,2,128,1,bfloat16,bfloat16,0,0.04819999933242798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,2,128,1,bfloat16,fp8,0,0.041812801361083986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,4,2,128,1,fp8,fp8,0,0.042268800735473636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,4,128,1,bfloat16,bfloat16,0,0.0408048003911972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,4,4,128,1,bfloat16,bfloat16,0,0.020340800285339355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,4,1,128,1,bfloat16,fp8,0,0.015459200739860535
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,4,128,1,bfloat16,fp8,0,0.03317280113697052
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,4,128,1,fp8,fp8,0,0.03307999968528748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,1,128,1,bfloat16,fp8,0,0.02319519966840744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,1,128,1,fp8,fp8,0,0.02343360036611557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,2,128,1,bfloat16,bfloat16,0,0.02382880002260208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,2,128,1,bfloat16,fp8,0,0.02343360036611557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,2,128,1,fp8,fp8,0,0.023180800676345825
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,4,128,1,bfloat16,bfloat16,0,0.02045599967241287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,4,128,1,bfloat16,fp8,0,0.020319999754428865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,4,128,1,fp8,fp8,0,0.020636799931526183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,1,128,1,bfloat16,bfloat16,0,0.015041600167751312
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,1,128,1,bfloat16,fp8,0,0.016220800578594208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,1,128,1,fp8,fp8,0,0.015747199952602386
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,2,128,1,bfloat16,bfloat16,0,0.015777599811553956
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,2,128,1,bfloat16,fp8,0,0.015820799767971037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,1,128,1,bfloat16,bfloat16,0,0.012889599800109864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,4,128,1,bfloat16,bfloat16,0,0.014164799451828003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,4,2,128,1,fp8,fp8,0,0.016206400096416475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,4,128,1,fp8,fp8,0,0.014424000680446625
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,1,128,1,bfloat16,bfloat16,0,0.013804799318313599
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,4,2,128,1,fp8,fp8,0,0.013857600092887879
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,1,128,1,bfloat16,fp8,0,0.01443839967250824
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,2,128,1,bfloat16,bfloat16,0,0.014028799533843995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,2,128,1,bfloat16,fp8,0,0.014144000411033631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,2,128,1,fp8,fp8,0,0.014567999541759491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,4,128,1,bfloat16,bfloat16,0,0.01297599971294403
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,4,128,1,bfloat16,fp8,0,0.013540799915790557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,4,128,1,fp8,fp8,0,0.013715200126171112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,1,128,1,bfloat16,bfloat16,0,0.013060800731182098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,1,128,1,bfloat16,fp8,0,0.01369599997997284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,1,128,1,fp8,fp8,0,0.013510400056838989
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,2,128,1,bfloat16,bfloat16,0,0.013303999602794648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,2,128,1,bfloat16,fp8,0,0.013708800077438354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,4,2,128,1,fp8,fp8,0,0.013956800103187561
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,4,128,1,bfloat16,fp8,0,0.013145600259304047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,4,128,1,fp8,fp8,0,0.01348000019788742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,1,128,1,bfloat16,bfloat16,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,1,128,1,bfloat16,fp8,0,0.01316159963607788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,1,128,1,fp8,fp8,0,0.01329600065946579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,2,128,1,bfloat16,bfloat16,0,0.01279360055923462
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,2,128,1,bfloat16,fp8,0,0.013316799700260163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,4,4,128,1,bfloat16,bfloat16,0,0.012067200243473053
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,2,128,1,fp8,fp8,0,0.013115200400352477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,4,128,1,bfloat16,bfloat16,0,0.012065599858760833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,4,128,1,bfloat16,fp8,0,0.01263200044631958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,4,128,1,fp8,fp8,0,0.012824000418186187
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,1,128,1,bfloat16,bfloat16,0,0.012174399942159653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,1,128,1,bfloat16,fp8,0,0.012744000554084778
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,1,128,1,fp8,fp8,0,0.012889599800109864
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,2,128,1,bfloat16,bfloat16,0,0.012169600278139115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,2,128,1,bfloat16,fp8,0,0.01268640011548996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,4,2,128,1,fp8,fp8,0,0.012913599610328674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,4,128,1,bfloat16,bfloat16,0,0.011832000315189361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,4,128,1,bfloat16,fp8,0,0.01239679977297783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,4,128,1,fp8,fp8,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,1,128,1,bfloat16,fp8,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,1,128,1,fp8,fp8,0,0.012699200212955475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,2,128,1,bfloat16,bfloat16,0,0.011948800086975098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,2,128,1,bfloat16,fp8,0,0.012734399735927581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,2,128,1,fp8,fp8,0,0.012545600533485413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,4,128,1,bfloat16,bfloat16,0,0.011753600090742111
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,4,128,1,bfloat16,fp8,0,0.01239359974861145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,4,128,1,fp8,fp8,0,0.012534399330615998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,1,128,1,bfloat16,bfloat16,0,0.011959999799728394
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,1,128,1,bfloat16,fp8,0,0.012620800733566284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,1,128,1,fp8,fp8,0,0.012649600207805634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,2,128,1,bfloat16,bfloat16,0,0.011966399848461151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,2,128,1,bfloat16,fp8,0,0.012639999389648438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,4,2,128,1,fp8,fp8,0,0.01263359934091568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,1,128,1,bfloat16,bfloat16,0,0.023689599335193635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,1,128,1,bfloat16,fp8,0,0.025537601113319396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,1,128,1,fp8,fp8,0,0.025275200605392456
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,2,128,1,bfloat16,bfloat16,0,0.03227680027484894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,2,128,1,bfloat16,fp8,0,0.03379679918289184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,4,2,128,1,fp8,fp8,0,0.03404799997806549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,4,128,1,bfloat16,bfloat16,0,0.028332799673080444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,4,1,128,1,bfloat16,bfloat16,0,0.022361600399017335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,4,128,1,bfloat16,fp8,0,0.030630400776863097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,4,128,1,fp8,fp8,0,0.03030399978160858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,1,128,1,bfloat16,bfloat16,0,0.01687840074300766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,2,128,1,bfloat16,bfloat16,0,0.019991999864578246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,2,128,1,bfloat16,fp8,0,0.021568000316619873
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,2,128,1,fp8,fp8,0,0.021505600214004515
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,4,128,1,bfloat16,bfloat16,0,0.01849119961261749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,4,128,1,bfloat16,fp8,0,0.019475199282169342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,4,128,1,fp8,fp8,0,0.019467200338840484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,4,128,1,bfloat16,fp8,0,0.014644800126552582
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,1,128,1,bfloat16,bfloat16,0,0.014326399564743042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,4,1,128,1,fp8,fp8,0,0.014139199256896972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,1,128,1,fp8,fp8,0,0.015046399831771851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,2,128,1,bfloat16,bfloat16,0,0.014219200611114502
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,2,128,1,fp8,fp8,0,0.015265600383281707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,4,128,1,bfloat16,bfloat16,0,0.01369439959526062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,4,128,1,bfloat16,fp8,0,0.014179199934005737
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,4,128,1,fp8,fp8,0,0.014107200503349304
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,1,128,1,bfloat16,bfloat16,0,0.013604800403118133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,1,128,1,bfloat16,fp8,0,0.014305600523948669
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,4,4,128,1,bfloat16,bfloat16,0,0.012780800461769104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,1,128,1,fp8,fp8,0,0.014230400323867798
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,2,128,1,bfloat16,fp8,0,0.01393280029296875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,2,128,1,bfloat16,bfloat16,0,0.013129599392414093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,4,2,128,1,fp8,fp8,0,0.013990400731563568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,4,128,1,bfloat16,bfloat16,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,4,128,1,bfloat16,fp8,0,0.013507199287414551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,4,128,1,fp8,fp8,0,0.01348000019788742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,1,128,1,bfloat16,bfloat16,0,0.012860800325870513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,1,128,1,bfloat16,fp8,0,0.013347199559211731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,1,128,1,fp8,fp8,0,0.013505600392818451
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,2,128,1,bfloat16,bfloat16,0,0.012542399764060973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,2,128,1,bfloat16,fp8,0,0.013459199666976928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,4,2,128,1,fp8,fp8,0,0.013368000090122224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,4,128,1,bfloat16,bfloat16,0,0.012380799651145935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,4,128,1,bfloat16,fp8,0,0.013353599607944489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,4,128,1,fp8,fp8,0,0.013012799620628356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,1,128,1,bfloat16,bfloat16,0,0.012488000094890594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,4,1,128,1,bfloat16,bfloat16,0,0.011849600076675414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,1,128,1,bfloat16,fp8,0,0.012887999415397644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,1,128,1,fp8,fp8,0,0.013153600692749023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,2,128,1,bfloat16,bfloat16,0,0.011947199702262878
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,2,128,1,bfloat16,fp8,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,4,2,128,1,fp8,fp8,0,0.01308799982070923
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,4,128,1,bfloat16,bfloat16,0,0.011745599657297134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,4,128,1,bfloat16,fp8,0,0.012665599584579468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,4,128,1,fp8,fp8,0,0.012700800597667695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,1,128,1,bfloat16,bfloat16,0,0.01186719983816147
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,1,128,1,bfloat16,fp8,0,0.012838399410247803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,1,128,1,fp8,fp8,0,0.01287200003862381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,2,128,1,bfloat16,bfloat16,0,0.011856000125408172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,2,128,1,bfloat16,fp8,0,0.012700800597667695
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,4,2,128,1,fp8,fp8,0,0.01268479973077774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,4,128,1,bfloat16,bfloat16,0,0.011715199798345566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,4,128,1,bfloat16,fp8,0,0.012558400630950928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,4,128,1,fp8,fp8,0,0.012600000202655792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,1,128,1,bfloat16,bfloat16,0,0.011905600130558015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,1,128,1,bfloat16,fp8,0,0.012736000120639801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,1,128,1,fp8,fp8,0,0.01279039978981018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,2,128,1,bfloat16,bfloat16,0,0.011750400066375732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,2,128,1,bfloat16,fp8,0,0.012492799758911132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,4,2,128,1,fp8,fp8,0,0.012755200266838074
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,4,128,1,bfloat16,bfloat16,0,0.011584000289440155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,4,128,1,bfloat16,fp8,0,0.012427199631929398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,4,128,1,fp8,fp8,0,0.012577599287033081
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,1,128,1,bfloat16,bfloat16,0,0.0117296002805233
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,1,128,1,bfloat16,fp8,0,0.012697599828243256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,1,128,1,fp8,fp8,0,0.012798400223255157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,2,128,1,bfloat16,bfloat16,0,0.011575999855995178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,2,128,1,bfloat16,fp8,0,0.012510399520397186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,4,2,128,1,fp8,fp8,0,0.012582400441169738
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,1,128,1,bfloat16,bfloat16,0,0.021719999611377716
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,1,128,1,bfloat16,fp8,0,0.023612800240516662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,1,128,1,fp8,fp8,0,0.024315199255943297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,2,128,1,bfloat16,bfloat16,0,0.02924799919128418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,2,128,1,bfloat16,fp8,0,0.032385599613189694
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,4,2,128,1,fp8,fp8,0,0.03271040022373199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,4,128,1,bfloat16,bfloat16,0,0.02720159888267517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,4,128,1,bfloat16,fp8,0,0.030067199468612672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,1,128,1,bfloat16,fp8,0,0.017129600048065186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,1,128,1,bfloat16,bfloat16,0,0.015255999565124512
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,1,128,1,bfloat16,fp8,0,0.015943999588489532
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,1,128,1,fp8,fp8,0,0.015878400206565856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,2,128,1,bfloat16,bfloat16,0,0.019420799612998963
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,2,128,1,bfloat16,fp8,0,0.020448000729084016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,2,128,1,fp8,fp8,0,0.020284800231456755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,4,128,1,bfloat16,bfloat16,0,0.018111999332904815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,4,128,1,bfloat16,fp8,0,0.019307200610637665
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,1,128,1,bfloat16,bfloat16,0,0.013896000385284425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,4,128,1,fp8,fp8,0,0.01947360038757324
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,1,128,1,bfloat16,fp8,0,0.014774399995803832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,1,128,1,fp8,fp8,0,0.014363199472427368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,2,128,1,bfloat16,bfloat16,0,0.014207999408245086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,2,128,1,bfloat16,fp8,0,0.01475519984960556
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,4,2,128,1,fp8,fp8,0,0.014803199470043183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,4,128,1,bfloat16,bfloat16,0,0.013308799266815186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,4,128,1,bfloat16,fp8,0,0.013950400054454803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,4,128,1,fp8,fp8,0,0.014091199636459351
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,1,128,1,bfloat16,bfloat16,0,0.013089600205421447
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,1,128,1,bfloat16,fp8,0,0.013860799372196198
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,1,128,1,fp8,fp8,0,0.013892799615859985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,1,128,1,bfloat16,fp8,0,0.015263999998569488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,2,128,1,bfloat16,fp8,0,0.013708800077438354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,2,128,1,fp8,fp8,0,0.013911999762058258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,4,128,1,bfloat16,fp8,0,0.01311040073633194
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,4,128,1,bfloat16,bfloat16,0,0.012417600303888322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,4,2,128,1,bfloat16,fp8,0,0.014601600170135499
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,1,128,1,bfloat16,bfloat16,0,0.012214399874210358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,1,128,1,bfloat16,fp8,0,0.012995199859142303
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,1,128,1,fp8,fp8,0,0.012768000364303589
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,2,128,1,bfloat16,fp8,0,0.013007999956607818
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,2,128,1,fp8,fp8,0,0.012809599936008453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,4,128,1,bfloat16,bfloat16,0,0.012273599952459335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,4,128,1,bfloat16,fp8,0,0.012759999930858612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,4,128,1,fp8,fp8,0,0.012940800189971924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,1,128,1,bfloat16,bfloat16,0,0.011793600022792816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,1,128,1,bfloat16,fp8,0,0.012611199915409089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,1,128,1,fp8,fp8,0,0.012620800733566284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,2,128,1,bfloat16,bfloat16,0,0.011902400106191636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,2,128,1,bfloat16,fp8,0,0.012646399438381195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,4,2,128,1,fp8,fp8,0,0.012680000066757202
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,4,128,1,bfloat16,bfloat16,0,0.011751999706029892
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,4,128,1,bfloat16,fp8,0,0.01250240057706833
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,4,128,1,fp8,fp8,0,0.012443199753761292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,1,128,1,bfloat16,bfloat16,0,0.011640000343322753
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,1,128,1,bfloat16,fp8,0,0.012414400279521943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,1,128,1,fp8,fp8,0,0.012414400279521943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,2,128,1,bfloat16,bfloat16,0,0.011721599847078323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,2,128,1,bfloat16,fp8,0,0.012444800138473511
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,4,2,128,1,fp8,fp8,0,0.012542399764060973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,4,128,1,bfloat16,bfloat16,0,0.011598400026559829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,4,128,1,bfloat16,fp8,0,0.012468799948692322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,4,128,1,fp8,fp8,0,0.012478400021791458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,1,128,1,bfloat16,bfloat16,0,0.01165120005607605
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,1,128,1,bfloat16,fp8,0,0.012412799894809723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,1,128,1,fp8,fp8,0,0.012324800342321396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,2,128,1,bfloat16,bfloat16,0,0.011559999734163284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,2,128,1,bfloat16,fp8,0,0.012336000055074691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,4,2,128,1,fp8,fp8,0,0.012334399670362473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,4,128,1,bfloat16,bfloat16,0,0.011540800333023071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,4,4,128,1,fp8,fp8,0,0.029875200986862183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,4,128,1,fp8,fp8,0,0.012449599802494049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,1,128,1,bfloat16,bfloat16,0,0.011748799681663513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,1,128,1,bfloat16,fp8,0,0.01263359934091568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,1,128,1,fp8,fp8,0,0.012639999389648438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,2,128,1,bfloat16,bfloat16,0,0.011806400120258331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,2,128,1,bfloat16,fp8,0,0.01247360035777092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,2,128,1,fp8,fp8,0,0.01255040019750595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,4,1,128,1,bfloat16,bfloat16,0,0.021563200652599333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,4,2,128,1,bfloat16,bfloat16,0,0.013118399679660797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,4,1,128,1,bfloat16,fp8,0,0.02321600019931793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,4,1,128,1,fp8,fp8,0,0.022545599937438966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,4,128,1,fp8,fp8,0,0.013166399300098419
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,4,2,128,1,bfloat16,bfloat16,0,0.02850080132484436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,4,2,128,1,bfloat16,fp8,0,0.031070399284362792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,4,2,128,1,fp8,fp8,0,0.031079998612403868
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,4,4,128,1,bfloat16,fp8,0,0.028948798775672913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,4,4,128,1,fp8,fp8,0,0.029131200909614564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,4,1,128,1,bfloat16,bfloat16,0,0.014815999567508698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,4,1,128,1,bfloat16,fp8,0,0.01563200056552887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,4,1,128,1,fp8,fp8,0,0.01576319932937622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,4,2,128,1,bfloat16,bfloat16,0,0.01876640021800995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,4,2,128,1,bfloat16,fp8,0,0.02001120001077652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,4,2,128,1,fp8,fp8,0,0.019888000190258028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,4,4,128,1,bfloat16,bfloat16,0,0.01720000058412552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,4,4,128,1,bfloat16,fp8,0,0.018545599281787874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,4,4,128,1,fp8,fp8,0,0.01868959963321686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,4,1,128,1,bfloat16,bfloat16,0,0.013439999520778656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,4,1,128,1,bfloat16,fp8,0,0.01414559930562973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,4,1,128,1,fp8,fp8,0,0.014168000221252442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,4,2,128,1,bfloat16,bfloat16,0,0.013331200182437896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,4,2,128,1,bfloat16,fp8,0,0.014350399374961853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,4,4,128,1,bfloat16,bfloat16,0,0.012972800433635712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,4,4,128,1,bfloat16,fp8,0,0.013836799561977387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,4,4,128,1,fp8,fp8,0,0.013801600039005279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,4,1,128,1,bfloat16,bfloat16,0,0.012788799405097962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,4,1,128,1,fp8,fp8,0,0.013512000441551208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,4,2,128,1,bfloat16,bfloat16,0,0.012798400223255157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,4,2,128,1,bfloat16,fp8,0,0.013551999628543854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,4,2,128,1,fp8,fp8,0,0.013604800403118133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,4,4,128,1,bfloat16,bfloat16,0,0.012438400089740754
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,4,4,128,1,bfloat16,fp8,0,0.013204799592494964
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,4,4,128,1,fp8,fp8,0,0.01318880021572113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,4,1,128,1,bfloat16,bfloat16,0,0.012038400024175644
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,4,1,128,1,bfloat16,fp8,0,0.01284479945898056
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,4,1,128,1,fp8,fp8,0,0.01279519945383072
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,4,2,128,1,bfloat16,bfloat16,0,0.01223199963569641
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,4,1,128,1,fp8,fp8,0,0.016739200055599212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,4,2,128,1,bfloat16,fp8,0,0.012961600720882416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,4,2,128,1,fp8,fp8,0,0.01297920048236847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,4,4,128,1,bfloat16,bfloat16,0,0.011790399998426437
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,4,4,128,1,bfloat16,fp8,0,0.012783999741077422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,4,4,128,1,fp8,fp8,0,0.012620800733566284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,4,1,128,1,bfloat16,bfloat16,0,0.011894399672746659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,4,1,128,1,bfloat16,fp8,0,0.012467200309038163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,4,1,128,1,fp8,fp8,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,4,2,128,1,bfloat16,bfloat16,0,0.011819200217723846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,4,2,128,1,bfloat16,fp8,0,0.012902399897575379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,4,2,128,1,fp8,fp8,0,0.012488000094890594
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,4,4,128,1,bfloat16,bfloat16,0,0.011899200081825257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,4,4,128,1,bfloat16,fp8,0,0.012331199645996094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,4,4,128,1,fp8,fp8,0,0.012715199589729309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,4,1,128,1,bfloat16,bfloat16,0,0.011379200220108032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,4,1,128,1,bfloat16,fp8,0,0.012399999797344208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,4,1,128,1,fp8,fp8,0,0.012107200175523757
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,4,2,128,1,bfloat16,bfloat16,0,0.011540800333023071
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,4,2,128,1,bfloat16,fp8,0,0.012435200065374375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,4,2,128,1,fp8,fp8,0,0.012145599722862244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,4,4,128,1,bfloat16,bfloat16,0,0.011519999802112579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,4,4,128,1,bfloat16,fp8,0,0.012377600371837615
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,4,1,128,1,bfloat16,bfloat16,0,0.011556799709796905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,4,1,128,1,bfloat16,fp8,0,0.012247999757528305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,4,1,128,1,fp8,fp8,0,0.012337599694728852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,4,2,128,1,bfloat16,bfloat16,0,0.012363199889659882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,4,2,128,1,bfloat16,bfloat16,0,0.011559999734163284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,4,2,128,1,bfloat16,fp8,0,0.012399999797344208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,4,2,128,1,fp8,fp8,0,0.012491200119256973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,4,4,128,1,bfloat16,bfloat16,0,0.011537600308656693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,4,4,128,1,bfloat16,fp8,0,0.012404800206422806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,4,4,128,1,fp8,fp8,0,0.012462399899959564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,4,1,128,1,bfloat16,bfloat16,0,0.01159999966621399
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,4,1,128,1,bfloat16,fp8,0,0.012356799840927125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,4,1,128,1,fp8,fp8,0,0.012323199957609176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,4,2,128,1,bfloat16,bfloat16,0,0.011575999855995178
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,4,2,128,1,bfloat16,fp8,0,0.01242400035262108
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,4,2,128,1,fp8,fp8,0,0.01228799968957901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,4,2,128,1,fp8,fp8,0,0.014316800236701965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,4,1,128,1,bfloat16,fp8,0,0.01348000019788742
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,2,1,128,1,bfloat16,fp8,0,1.6056671142578125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,2,128,1,bfloat16,bfloat16,0,0.8899295806884766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,4,4,128,1,bfloat16,fp8,0,0.012430399656295776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,2,1,128,1,fp8,fp8,0,1.6078624725341797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,2,128,1,bfloat16,fp8,0,0.8319696426391602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,2,128,1,fp8,fp8,0,0.8657967567443847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,1,128,1,bfloat16,bfloat16,0,0.8617088317871093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,4,4,128,1,fp8,fp8,0,0.012225600332021714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,4,4,128,1,bfloat16,bfloat16,0,0.02630400061607361
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,2,128,1,bfloat16,bfloat16,0,0.4513807773590088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,1,128,1,fp8,fp8,0,0.811185646057129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,2,128,1,fp8,fp8,0,0.4262239933013916
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,1,128,1,bfloat16,bfloat16,0,0.432696008682251
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,1,128,1,bfloat16,fp8,0,0.42356319427490235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,2,128,1,bfloat16,bfloat16,0,0.22399520874023438
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,1,128,1,fp8,fp8,0,0.4084512233734131
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,2,128,1,bfloat16,fp8,0,0.21974239349365235
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,2,128,1,fp8,fp8,0,0.22324800491333008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,1,128,1,bfloat16,bfloat16,0,0.21700000762939453
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,1,128,1,bfloat16,fp8,0,0.21415679454803466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,2,1,128,1,fp8,fp8,0,0.2178879976272583
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,2,128,1,bfloat16,bfloat16,0,0.5653679847717286
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,2,1,128,1,bfloat16,bfloat16,0,1.013491153717041
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,2,1,128,1,bfloat16,fp8,0,0.9512319564819336
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,12288,2,1,128,1,fp8,fp8,0,0.9486528396606445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16384,2,1,128,1,bfloat16,bfloat16,0,1.7407312393188477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,2,128,1,fp8,fp8,0,0.5024864196777343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,1,128,1,bfloat16,bfloat16,0,0.5400063991546631
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,1,128,1,bfloat16,fp8,0,0.4942080020904541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,2,128,1,bfloat16,bfloat16,0,0.31201920509338377
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,2,128,1,bfloat16,fp8,0,0.26714239120483396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,2,128,1,fp8,fp8,0,0.28465120792388915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,1,128,1,bfloat16,bfloat16,0,0.27284960746765136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,1,128,1,bfloat16,fp8,0,0.2669951915740967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,2,1,128,1,fp8,fp8,0,0.26620481014251707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,2,128,1,bfloat16,bfloat16,0,0.18370720148086547
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,2,128,1,fp8,fp8,0,0.1649456024169922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,1,128,1,bfloat16,bfloat16,0,0.16851359605789185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,1,128,1,bfloat16,fp8,0,0.17632960081100463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,2,1,128,1,bfloat16,fp8,0,0.8138640403747559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,1,128,1,fp8,fp8,0,0.16593120098114014
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,2,2,128,1,bfloat16,fp8,0,0.42191519737243655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,2,128,1,bfloat16,bfloat16,0,0.4192768096923828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,2,1,128,1,bfloat16,bfloat16,0,0.7220176219940185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,2,1,128,1,fp8,fp8,0,0.6817808151245117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,10240,2,1,128,1,bfloat16,fp8,0,0.7360943794250489
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,2,128,1,bfloat16,fp8,0,0.35873119831085204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,2,128,1,fp8,fp8,0,0.3643791913986206
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,1,128,1,bfloat16,bfloat16,0,0.3741391897201538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,2,128,1,bfloat16,bfloat16,0,0.24280800819396972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,1,128,1,bfloat16,fp8,0,0.35728960037231444
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,2,128,1,bfloat16,fp8,0,0.1929792046546936
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,2,1,128,1,fp8,fp8,0,0.36463680267333987
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,2,128,1,fp8,fp8,0,0.2018496036529541
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,1,128,1,bfloat16,bfloat16,0,0.19650559425354003
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,1,128,1,bfloat16,fp8,0,0.18646240234375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,2,128,1,bfloat16,bfloat16,0,0.14548799991607667
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,2,1,128,1,fp8,fp8,0,0.1916159987449646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,2,128,1,bfloat16,fp8,0,0.14995839595794677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,2,128,1,fp8,fp8,0,0.13691999912261962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,1,128,1,bfloat16,bfloat16,0,0.14174879789352418
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,1,128,1,bfloat16,fp8,0,0.1377568006515503
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,2,1,128,1,fp8,fp8,0,0.13871840238571168
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,2,128,1,bfloat16,fp8,0,0.49691038131713866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,2,128,1,bfloat16,fp8,0,0.45115041732788086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,2,128,1,fp8,fp8,0,0.44571361541748045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,2,1,128,1,bfloat16,bfloat16,0,0.9367808341979981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,2,1,128,1,bfloat16,fp8,0,0.8665984153747559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,8192,2,1,128,1,fp8,fp8,0,0.8771792411804199
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,2,1,128,1,fp8,fp8,0,0.5000688076019287
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,1,128,1,bfloat16,bfloat16,0,0.4666560173034668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,2,128,1,bfloat16,bfloat16,0,0.24800639152526854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,2,128,1,bfloat16,fp8,0,0.2315999984741211
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,2,128,1,fp8,fp8,0,0.22810559272766112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,1,128,1,bfloat16,fp8,0,0.44293918609619143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,1,128,1,fp8,fp8,0,0.4827871799468994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,1,128,1,bfloat16,bfloat16,0,0.23907039165496827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,2,128,1,bfloat16,bfloat16,0,0.1278112053871155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,1,128,1,bfloat16,fp8,0,0.22559680938720703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,2,128,1,bfloat16,fp8,0,0.11871360540390015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,2,1,128,1,fp8,fp8,0,0.22811200618743896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,2,128,1,fp8,fp8,0,0.12498719692230224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,1,128,1,bfloat16,fp8,0,0.11714880466461182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,2,128,1,bfloat16,bfloat16,0,0.11361600160598755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,2,128,1,bfloat16,fp8,0,0.11115360260009766
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,2,128,1,fp8,fp8,0,0.11917120218276978
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,1,128,1,bfloat16,bfloat16,0,0.11174399852752685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,1,128,1,bfloat16,fp8,0,0.11081919670104981
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,2,1,128,1,fp8,fp8,0,0.11181600093841552
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,2,128,1,bfloat16,bfloat16,0,0.3301408052444458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,2,1,128,1,bfloat16,bfloat16,0,0.5942815780639649
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,2,1,128,1,bfloat16,fp8,0,0.5554927825927735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,6144,2,1,128,1,fp8,fp8,0,0.5549280166625976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,2,128,1,bfloat16,fp8,0,0.30389440059661865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,2,128,1,fp8,fp8,0,0.2993920087814331
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,1,128,1,bfloat16,bfloat16,0,0.31157920360565183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,1,128,1,bfloat16,fp8,0,0.30156960487365725
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,2,128,1,bfloat16,bfloat16,0,0.17914400100708008
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,2,1,128,1,fp8,fp8,0,0.2968816041946411
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,2,128,1,bfloat16,fp8,0,0.1660879969596863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,2,128,1,fp8,fp8,0,0.16359200477600097
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,1,128,1,bfloat16,bfloat16,0,0.17077120542526245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,1,128,1,bfloat16,fp8,0,0.16456160545349122
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,2,1,128,1,fp8,fp8,0,0.16195039749145507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,2,2,128,1,bfloat16,bfloat16,0,0.489086389541626
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,2,128,1,bfloat16,fp8,0,0.09425439834594726
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,2,128,1,fp8,fp8,0,0.092849600315094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,1,128,1,bfloat16,fp8,0,0.09389920234680176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,1,128,1,fp8,fp8,0,0.09247519969940185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,2,128,1,bfloat16,bfloat16,0,0.07120479941368103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,2,128,1,bfloat16,fp8,0,0.06840479969978333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,2,128,1,fp8,fp8,0,0.06930400133132934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,1,128,1,bfloat16,bfloat16,0,0.0610256016254425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,1,128,1,bfloat16,fp8,0,0.06092159748077393
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,2,1,128,1,fp8,fp8,0,0.06036800146102905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,2,2,128,1,bfloat16,fp8,0,0.1635151982307434
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,1,128,1,bfloat16,bfloat16,0,0.12197439670562744
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,2,1,128,1,fp8,fp8,0,0.1185744047164917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,2,1,128,1,bfloat16,fp8,0,0.5336495876312256
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,2,1,128,1,bfloat16,bfloat16,0,0.5655183792114258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,2,128,1,bfloat16,bfloat16,0,0.3233776092529297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,2,128,1,bfloat16,fp8,0,0.2934159994125366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,4096,2,1,128,1,fp8,fp8,0,0.5346672058105468
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,2,128,1,fp8,fp8,0,0.29238879680633545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,2,128,1,bfloat16,bfloat16,0,0.17719039916992188
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,1,128,1,bfloat16,fp8,0,0.28999838829040525
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,1,128,1,fp8,fp8,0,0.2890511989593506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,2,128,1,bfloat16,fp8,0,0.16347839832305908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,2,128,1,fp8,fp8,0,0.16054400205612182
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,1,128,1,bfloat16,bfloat16,0,0.16931040287017823
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,2,128,1,bfloat16,bfloat16,0,0.10552639961242676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,1,128,1,fp8,fp8,0,0.16151679754257203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,2,128,1,bfloat16,fp8,0,0.09789119958877564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,2,128,1,fp8,fp8,0,0.09644479751586914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,1,128,1,bfloat16,bfloat16,0,0.10157120227813721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,1,128,1,bfloat16,fp8,0,0.09820960164070129
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,2,128,1,bfloat16,bfloat16,0,0.06733919978141785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,2,128,1,bfloat16,fp8,0,0.06718239784240723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,2,1,128,1,fp8,fp8,0,0.0963919997215271
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,2,128,1,fp8,fp8,0,0.06619840264320373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,1,128,1,bfloat16,bfloat16,0,0.06709120273590088
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,1,128,1,bfloat16,fp8,0,0.06592000126838685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,2,1,128,1,fp8,fp8,0,0.06645280122756958
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,2,128,1,bfloat16,fp8,0,0.04415520131587982
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,2,128,1,bfloat16,bfloat16,0,0.100547194480896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,2,128,1,fp8,fp8,0,0.04483680129051208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,1,128,1,bfloat16,fp8,0,0.04480000138282776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,1,128,1,fp8,fp8,0,0.04472799897193909
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,2,1,128,1,bfloat16,bfloat16,0,0.09619680047035217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,2,1,128,1,bfloat16,bfloat16,0,0.3535392045974731
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,2,1,128,1,bfloat16,fp8,0,0.34219520092010497
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,3072,2,1,128,1,fp8,fp8,0,0.34219839572906496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,2,128,1,bfloat16,bfloat16,0,0.21030080318450928
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,2,128,1,bfloat16,fp8,0,0.19027520418167115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,2,128,1,fp8,fp8,0,0.19143999814987184
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,1,128,1,bfloat16,bfloat16,0,0.1939311981201172
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,1,128,1,bfloat16,fp8,0,0.1861631989479065
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,2,128,1,bfloat16,bfloat16,0,0.11940959692001343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,2,128,1,bfloat16,fp8,0,0.10921280384063721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,2,128,1,fp8,fp8,0,0.11005439758300781
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,1,128,1,bfloat16,bfloat16,0,0.11228799819946289
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,1,128,1,bfloat16,fp8,0,0.10758240222930908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,2,1,128,1,fp8,fp8,0,0.10607359409332276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,2,1,128,1,bfloat16,bfloat16,0,0.30137441158294676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,2,128,1,bfloat16,bfloat16,0,0.0702895998954773
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,2,128,1,bfloat16,fp8,0,0.06662399768829345
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,1,128,1,bfloat16,fp8,0,0.0654367983341217
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,1,128,1,bfloat16,bfloat16,0,0.06562079787254334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,2,128,1,fp8,fp8,0,0.06711360216140747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,2,1,128,1,bfloat16,fp8,0,0.1611232042312622
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,2,128,1,bfloat16,bfloat16,0,0.053630399703979495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,2,1,128,1,fp8,fp8,0,0.06499519944190979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,2,128,1,bfloat16,fp8,0,0.051830399036407473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,2,128,1,fp8,fp8,0,0.051344001293182374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,1,128,1,bfloat16,bfloat16,0,0.04881759881973267
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,1,128,1,bfloat16,fp8,0,0.049507200717926025
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,2,128,1,bfloat16,bfloat16,0,0.031446400284767154
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,2,1,128,1,fp8,fp8,0,0.04970560073852539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,2,128,1,bfloat16,fp8,0,0.03226400017738342
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,1,128,1,bfloat16,bfloat16,0,0.03148640096187592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,2,128,1,fp8,fp8,0,0.03252480030059814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,1,128,1,fp8,fp8,0,0.03208320140838623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,2,128,1,bfloat16,bfloat16,0,0.04583359956741333
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,2,128,1,bfloat16,bfloat16,0,0.2150320053100586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,2,1,128,1,bfloat16,bfloat16,0,0.04438399970531463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,2,1,128,1,bfloat16,fp8,0,0.34621601104736327
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,2,1,128,1,fp8,fp8,0,0.3508016109466553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,2,128,1,bfloat16,fp8,0,0.19236479997634887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,2,128,1,fp8,fp8,0,0.19086400270462037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,1,128,1,bfloat16,bfloat16,0,0.1894703984260559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,1,128,1,bfloat16,fp8,0,0.1867632031440735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,2,128,1,bfloat16,bfloat16,0,0.12067199945449829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,2,128,1,bfloat16,fp8,0,0.107750403881073
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,2,128,1,fp8,fp8,0,0.10599039793014527
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,1,128,1,bfloat16,bfloat16,0,0.10953600406646728
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,1,128,1,bfloat16,fp8,0,0.10473120212554932
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,2,1,128,1,fp8,fp8,0,0.10253599882125855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,2,128,1,bfloat16,fp8,0,0.06567999720573425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,2,128,1,fp8,fp8,0,0.06439200043678284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,1,128,1,bfloat16,bfloat16,0,0.06648640036582946
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,1,128,1,bfloat16,fp8,0,0.06339359879493714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,1,128,1,fp8,fp8,0,0.06471199989318847
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,2,128,1,bfloat16,bfloat16,0,0.041326400637626645
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,2,128,1,bfloat16,fp8,0,0.041443198919296265
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,2,128,1,fp8,fp8,0,0.040633600950241086
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,2,1,128,1,fp8,fp8,0,0.18741919994354247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,1,128,1,bfloat16,bfloat16,0,0.04147680103778839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,1,128,1,bfloat16,fp8,0,0.040648001432418826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,2,1,128,1,fp8,fp8,0,0.04095999896526337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,2,128,1,bfloat16,bfloat16,0,0.032779198884963986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,2,128,1,bfloat16,fp8,0,0.032471999526023865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,2,128,1,fp8,fp8,0,0.03274720013141632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,1,128,1,bfloat16,bfloat16,0,0.03240480124950409
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,1,128,1,bfloat16,fp8,0,0.03249120116233826
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,2,1,128,1,fp8,fp8,0,0.03272640109062195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,2,128,1,bfloat16,bfloat16,0,0.023401600122451783
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,2,128,1,bfloat16,fp8,0,0.024027200043201448
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,2,128,1,fp8,fp8,0,0.024345600605010988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,1,128,1,bfloat16,bfloat16,0,0.022945599257946016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,1,128,1,bfloat16,fp8,0,0.02399359941482544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,2,1,128,1,fp8,fp8,0,0.024196800589561463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,2,1,128,1,bfloat16,bfloat16,0,0.2349776029586792
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,2,1,128,1,bfloat16,fp8,0,0.2272336006164551
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1536,2,1,128,1,fp8,fp8,0,0.22767200469970703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,2,128,1,bfloat16,bfloat16,0,0.14606399536132814
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,2,1,128,1,bfloat16,fp8,0,0.03181760013103485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,2,128,1,fp8,fp8,0,0.12866079807281494
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,1,128,1,bfloat16,bfloat16,0,0.1292799949645996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,1,128,1,fp8,fp8,0,0.12454240322113037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,2,128,1,bfloat16,bfloat16,0,0.0822767972946167
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,2048,2,1,128,1,bfloat16,bfloat16,0,0.35589599609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,2,128,1,bfloat16,fp8,0,0.07539680004119872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,2,128,1,fp8,fp8,0,0.07591360211372375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,1,128,1,bfloat16,fp8,0,0.07187520265579224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,1,128,1,bfloat16,bfloat16,0,0.07533919811248779
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,2,1,128,1,fp8,fp8,0,0.07264320254325866
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,2,128,1,bfloat16,bfloat16,0,0.05119519829750061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,2,128,1,bfloat16,fp8,0,0.04830560088157654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,2,1,128,1,fp8,fp8,0,0.18824800252914428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,1,128,1,bfloat16,bfloat16,0,0.04589119851589203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,1,128,1,bfloat16,fp8,0,0.048089599609375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,2,128,1,bfloat16,bfloat16,0,0.04088639914989471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,2,128,1,bfloat16,fp8,0,0.04059999883174896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,2,128,1,fp8,fp8,0,0.040043199062347413
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,1,128,1,bfloat16,bfloat16,0,0.039299198985099794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,2,2,128,1,bfloat16,bfloat16,0,0.07359840273857117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,1,128,1,bfloat16,fp8,0,0.04008800089359284
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,2,1,128,1,fp8,fp8,0,0.04061119854450226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,2,128,1,bfloat16,bfloat16,0,0.026206400990486146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,2,128,1,bfloat16,fp8,0,0.02630079984664917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,2,128,1,fp8,fp8,0,0.026630398631095887
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,1,128,1,bfloat16,bfloat16,0,0.02489279955625534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,1,128,1,bfloat16,fp8,0,0.02637600004673004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,2,1,128,1,fp8,fp8,0,0.02664799988269806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,2,128,1,bfloat16,bfloat16,0,0.020839999616146087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,2,128,1,bfloat16,fp8,0,0.020414400100708007
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,2,128,1,fp8,fp8,0,0.020636799931526183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,1,128,1,bfloat16,bfloat16,0,0.020614400506019592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,1,128,1,bfloat16,fp8,0,0.020227199792861937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,2,1,128,1,fp8,fp8,0,0.020630399882793426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,2,1,128,1,bfloat16,bfloat16,0,0.25046238899230955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,2,1,128,1,bfloat16,fp8,0,0.24928319454193115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1024,2,1,128,1,fp8,fp8,0,0.2526063919067383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,2,128,1,bfloat16,fp8,0,0.14060319662094117
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,2,128,1,fp8,fp8,0,0.14283839464187623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,1,128,1,bfloat16,bfloat16,0,0.13477120399475098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,1,128,1,bfloat16,fp8,0,0.1346160054206848
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,1,128,1,fp8,fp8,0,0.1342560052871704
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,2,128,1,bfloat16,bfloat16,0,0.08792960047721862
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,2,128,1,bfloat16,fp8,0,0.12811039686203002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,2,128,1,bfloat16,fp8,0,0.07982239723205567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,2,128,1,fp8,fp8,0,0.08052800297737121
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,1,128,1,bfloat16,bfloat16,0,0.07692639827728272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,2,1,128,1,bfloat16,fp8,0,0.12553759813308715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,1,128,1,bfloat16,fp8,0,0.07585279941558838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,2,1,128,1,fp8,fp8,0,0.07733759880065919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,2,128,1,bfloat16,bfloat16,0,0.052750402688980104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,2,128,1,bfloat16,fp8,0,0.04836480021476745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,2,128,1,fp8,fp8,0,0.04824160039424896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,1,128,1,bfloat16,fp8,0,0.04679999947547912
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,1,128,1,bfloat16,bfloat16,0,0.04663360118865967
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,2,128,1,bfloat16,bfloat16,0,0.029183998703956604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,2,128,1,bfloat16,fp8,0,0.02996160089969635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,2,128,1,fp8,fp8,0,0.02980479896068573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,1,128,1,bfloat16,bfloat16,0,0.028763198852539064
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,1,128,1,bfloat16,fp8,0,0.02932640016078949
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,2,1,128,1,fp8,fp8,0,0.02972320020198822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,2,128,1,bfloat16,bfloat16,0,0.026161599159240722
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,2,128,1,fp8,fp8,0,0.04800800085067749
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,2,128,1,bfloat16,fp8,0,0.027249601483345032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,2,128,1,fp8,fp8,0,0.027423998713493346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,1,128,1,bfloat16,bfloat16,0,0.02606239914894104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,1,128,1,bfloat16,fp8,0,0.02731199860572815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,2,128,1,bfloat16,bfloat16,0,0.021372799575328828
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,2,1,128,1,fp8,fp8,0,0.04789279997348785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,2,128,1,fp8,fp8,0,0.02096800059080124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,1,128,1,bfloat16,bfloat16,0,0.021110400557518005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,1,128,1,bfloat16,fp8,0,0.02110559940338135
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,1,128,1,fp8,fp8,0,0.020975999534130096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,2,128,1,bfloat16,bfloat16,0,0.018376000225543976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,2,128,1,bfloat16,fp8,0,0.019886399805545806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,2,128,1,fp8,fp8,0,0.01966560035943985
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,1,128,1,bfloat16,bfloat16,0,0.018531200289726258
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,1,128,1,bfloat16,fp8,0,0.019985599815845488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,2,1,128,1,fp8,fp8,0,0.019750399887561797
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,2,1,128,1,bfloat16,bfloat16,0,0.17350720167160033
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,2,1,128,1,bfloat16,fp8,0,0.17999039888381957
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,512,2,1,128,1,fp8,fp8,0,0.17781440019607545
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,2,128,1,bfloat16,bfloat16,0,0.11855360269546508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,2,128,1,bfloat16,fp8,0,0.10415199995040894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,2,128,1,fp8,fp8,0,0.1034208059310913
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,1,128,1,bfloat16,bfloat16,0,0.09452959895133972
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,1,128,1,bfloat16,fp8,0,0.09794080257415771
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,2,1,128,1,fp8,fp8,0,0.09692320227622986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,2,128,1,bfloat16,bfloat16,0,0.0675104022026062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,2,128,1,bfloat16,fp8,0,0.060417598485946654
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,2,128,1,fp8,fp8,0,0.059721601009368894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,1,128,1,bfloat16,bfloat16,0,0.05469759702682495
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,1,128,1,bfloat16,fp8,0,0.05626400113105774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,2,1,128,1,fp8,fp8,0,0.05544319748878479
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,2,128,1,bfloat16,bfloat16,0,0.039643201231956485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,2,128,1,bfloat16,fp8,0,0.0350735992193222
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,2,128,1,fp8,fp8,0,0.034904000163078305
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,1,128,1,bfloat16,bfloat16,0,0.03208160102367401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,1,128,1,bfloat16,fp8,0,0.03364959955215454
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,2,2,128,1,bfloat16,bfloat16,0,0.15889919996261598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,2,1,128,1,fp8,fp8,0,0.0333871990442276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,2,128,1,bfloat16,bfloat16,0,0.020431999862194062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,2,128,1,bfloat16,fp8,0,0.021755200624465943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,1,128,1,bfloat16,bfloat16,0,0.020057600736618043
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,2,128,1,fp8,fp8,0,0.021784000098705292
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,1,128,1,bfloat16,fp8,0,0.021303999423980712
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,2,1,128,1,fp8,fp8,0,0.021481600403785706
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,2,128,1,bfloat16,bfloat16,0,0.01871040016412735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,2,128,1,bfloat16,fp8,0,0.020044800639152528
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,2,128,1,fp8,fp8,0,0.019857600331306458
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,1,128,1,bfloat16,bfloat16,0,0.018612800538539885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,1,128,1,bfloat16,fp8,0,0.019972799718379973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,2,1,128,1,fp8,fp8,0,0.01977120041847229
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,2,128,1,bfloat16,bfloat16,0,0.017936000227928163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,2,128,1,bfloat16,fp8,0,0.019655999541282655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,2,128,1,fp8,fp8,0,0.019449600577354433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,1,128,1,bfloat16,bfloat16,0,0.017371200025081635
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,1,128,1,bfloat16,fp8,0,0.019144000113010408
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,2,128,1,bfloat16,bfloat16,0,0.01730719953775406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,2,128,1,bfloat16,fp8,0,0.018961599469184874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,2,128,1,fp8,fp8,0,0.019064000248908995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,1,128,1,bfloat16,bfloat16,0,0.01724960058927536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,1,128,1,bfloat16,fp8,0,0.01881600022315979
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,2,1,128,1,fp8,fp8,0,0.01884479969739914
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,2,128,1,bfloat16,fp8,0,0.018855999410152435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,2,128,1,fp8,fp8,0,0.018849599361419677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,1,128,1,bfloat16,bfloat16,0,0.01695999950170517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,1,128,1,bfloat16,fp8,0,0.018648000061511995
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,1,128,1,fp8,fp8,0,0.018806399405002595
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,2,1,128,1,bfloat16,bfloat16,0,0.08679680228233337
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,2,1,128,1,fp8,fp8,0,0.046480000019073486
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,2,1,128,1,bfloat16,fp8,0,0.08665760159492493
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,256,2,1,128,1,fp8,fp8,0,0.08569599986076355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,2,128,1,bfloat16,fp8,0,0.05142880082130432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,2,128,1,bfloat16,bfloat16,0,0.06402720212936401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,2,128,1,fp8,fp8,0,0.05108320116996765
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,1,128,1,bfloat16,bfloat16,0,0.04912959933280945
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,1,128,1,bfloat16,fp8,0,0.04863840043544769
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,2,128,1,bfloat16,bfloat16,0,0.0349727988243103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,2,128,1,bfloat16,fp8,0,0.02874400019645691
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,2,1,128,1,fp8,fp8,0,0.027326399087905885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,2,128,1,fp8,fp8,0,0.028889599442481994
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,1,128,1,bfloat16,bfloat16,0,0.027427199482917785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,1,128,1,bfloat16,fp8,0,0.02871679961681366
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,2,128,1,bfloat16,bfloat16,0,0.01797440052032471
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,2,128,1,bfloat16,fp8,0,0.018374399840831758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,1,128,1,bfloat16,bfloat16,0,0.01762080043554306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,1,128,1,bfloat16,fp8,0,0.018243199586868285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,1,128,1,fp8,fp8,0,0.018276800215244294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,2,128,1,bfloat16,bfloat16,0,0.015800000727176668
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,2,128,1,bfloat16,fp8,0,0.01640959978103638
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,2,128,1,fp8,fp8,0,0.01629599928855896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,1,128,1,bfloat16,bfloat16,0,0.015275199711322785
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,1,128,1,bfloat16,fp8,0,0.016326400637626647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,2,1,128,1,fp8,fp8,0,0.01639360040426254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,2,128,1,bfloat16,bfloat16,0,0.015384000539779664
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,2,128,1,bfloat16,fp8,0,0.01582240015268326
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,2,128,1,fp8,fp8,0,0.015651200711727143
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,1,128,1,bfloat16,bfloat16,0,0.014532800018787383
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,1,128,1,bfloat16,fp8,0,0.015857599675655365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,2,1,128,1,fp8,fp8,0,0.015372799336910247
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,2,128,1,bfloat16,bfloat16,0,0.014300799369812012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,2,128,1,bfloat16,fp8,0,0.015414400398731232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,2,128,1,fp8,fp8,0,0.015326400101184846
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,2,1,128,1,fp8,fp8,0,0.01931840032339096
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,1,128,1,bfloat16,bfloat16,0,0.014120000600814819
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,1,128,1,bfloat16,fp8,0,0.015046399831771851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,2,2,128,1,bfloat16,bfloat16,0,0.01693760007619858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,2,1,128,1,fp8,fp8,0,0.01525759994983673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,2,128,1,bfloat16,bfloat16,0,0.014086399972438813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,2,128,1,bfloat16,fp8,0,0.01491519957780838
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,2,128,1,fp8,fp8,0,0.015044799447059632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,1,128,1,bfloat16,bfloat16,0,0.013910399377346038
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,1,128,1,bfloat16,fp8,0,0.015043200552463531
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,2,1,128,1,fp8,fp8,0,0.01496960073709488
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,2,128,1,bfloat16,bfloat16,0,0.013828800618648529
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,2,128,1,bfloat16,fp8,0,0.014740799367427827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,2,128,1,fp8,fp8,0,0.014934399724006652
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,1,128,1,bfloat16,fp8,0,0.014763200283050537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,1,128,1,bfloat16,bfloat16,0,0.013577599823474885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,2,1,128,1,fp8,fp8,0,0.015087999403476715
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,2,1,128,1,bfloat16,bfloat16,0,0.04873600006103516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,2,1,128,1,fp8,fp8,0,0.04686079919338226
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,2,128,1,bfloat16,bfloat16,0,0.033451199531555176
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,2,128,1,bfloat16,fp8,0,0.026180800795555115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,2,128,1,fp8,fp8,0,0.02630240023136139
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,2,2,128,1,bfloat16,fp8,0,0.021132799983024596
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,1,128,1,bfloat16,fp8,0,0.025809600949287415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,1,128,1,fp8,fp8,0,0.026177600026130676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,2,1,128,1,fp8,fp8,0,0.028457599878311157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,2,128,1,bfloat16,bfloat16,0,0.017161600291728973
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,2,128,1,bfloat16,fp8,0,0.01759520024061203
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,1,128,1,bfloat16,bfloat16,0,0.016972799599170686
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,2,128,1,fp8,fp8,0,0.017217600345611574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,1,128,1,bfloat16,fp8,0,0.017308799922466277
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,2,128,1,bfloat16,bfloat16,0,0.014659200608730317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,2,128,1,bfloat16,fp8,0,0.014996799826622009
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,2,128,1,fp8,fp8,0,0.014817599952220917
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,1,128,1,bfloat16,bfloat16,0,0.014316800236701965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,1,128,1,bfloat16,fp8,0,0.014827199280261993
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,2,1,128,1,fp8,fp8,0,0.014769600331783294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,2,128,1,bfloat16,bfloat16,0,0.013952000439167023
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,2,128,1,bfloat16,fp8,0,0.013574400544166565
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,2,128,1,fp8,fp8,0,0.013769599795341491
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,1,128,1,bfloat16,bfloat16,0,0.013505600392818451
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,1,128,1,bfloat16,fp8,0,0.013478399813175201
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,2,1,128,1,fp8,fp8,0,0.013963200151920319
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,2,128,1,bfloat16,bfloat16,0,0.0127920001745224
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,2,128,1,bfloat16,fp8,0,0.013201600313186646
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,2,128,1,fp8,fp8,0,0.01326880007982254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,1,128,1,bfloat16,bfloat16,0,0.012910400331020356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,1,128,1,bfloat16,fp8,0,0.013070400059223174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,2,1,128,1,fp8,fp8,0,0.013372799754142762
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,2,128,1,bfloat16,bfloat16,0,0.012345600128173827
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,2,128,1,bfloat16,fp8,0,0.013097600638866424
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,2,128,1,fp8,fp8,0,0.012822400033473968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,1,128,1,bfloat16,bfloat16,0,0.012476799637079239
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,1,128,1,bfloat16,fp8,0,0.012852799892425538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,2,1,128,1,fp8,fp8,0,0.01308320015668869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,2,128,1,bfloat16,bfloat16,0,0.012006399780511856
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,2,128,1,bfloat16,fp8,0,0.012902399897575379
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,2,128,1,fp8,fp8,0,0.012624000012874604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,1,128,1,bfloat16,bfloat16,0,0.012203200161457062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,1,128,1,bfloat16,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,2,1,128,1,fp8,fp8,0,0.012801599502563477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,2,128,1,bfloat16,bfloat16,0,0.011915200203657151
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,2,128,1,bfloat16,fp8,0,0.01265919953584671
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,2,128,1,fp8,fp8,0,0.012572799623012543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,1,128,1,bfloat16,bfloat16,0,0.012167999893426895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,1,128,1,bfloat16,fp8,0,0.012583999335765839
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,2,1,128,1,fp8,fp8,0,0.01276479959487915
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,2,1,128,1,bfloat16,bfloat16,0,0.02731359899044037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,2,1,128,1,bfloat16,fp8,0,0.027884799242019653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,64,2,1,128,1,fp8,fp8,0,0.02784320116043091
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,2,128,1,bfloat16,bfloat16,0,0.021107199788093566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,2,128,1,bfloat16,fp8,0,0.022436800599098205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,2,128,1,fp8,fp8,0,0.022470399737358093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,1,128,1,bfloat16,bfloat16,0,0.017574399709701538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,1,128,1,bfloat16,fp8,0,0.018326400220394133
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,2,1,128,1,fp8,fp8,0,0.01798879951238632
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,2,128,1,bfloat16,bfloat16,0,0.015046399831771851
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,2,128,1,bfloat16,fp8,0,0.015412800014019012
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,2,128,1,fp8,fp8,0,0.015595200657844543
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,1,128,1,bfloat16,bfloat16,0,0.014632000029087067
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,1,128,1,bfloat16,fp8,0,0.015465599298477174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,2,1,128,1,fp8,fp8,0,0.015363200008869171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,128,2,1,128,1,bfloat16,fp8,0,0.04610880017280579
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,2,128,1,bfloat16,fp8,0,0.013939200341701508
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,1,128,1,bfloat16,bfloat16,0,0.013449600338935852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,2,1,128,1,fp8,fp8,0,0.04830079972743988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,1,128,1,bfloat16,fp8,0,0.013969600200653076
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,2,1,128,1,bfloat16,bfloat16,0,0.02563680112361908
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,1,128,1,fp8,fp8,0,0.014379200339317322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,2,128,1,bfloat16,bfloat16,0,0.012742400169372559
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,2,128,1,fp8,fp8,0,0.013624000549316406
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,2,128,1,bfloat16,fp8,0,0.013199999928474426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,1,128,1,bfloat16,fp8,0,0.013494400680065155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,1,128,1,fp8,fp8,0,0.013033600151538849
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,2,2,128,1,fp8,fp8,0,0.018262399733066557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,2,128,1,bfloat16,bfloat16,0,0.012483199685811996
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,2,128,1,bfloat16,fp8,0,0.01282079964876175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,2,128,1,fp8,fp8,0,0.013036799430847169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,1,128,1,bfloat16,bfloat16,0,0.012585599720478059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,2,1,128,1,fp8,fp8,0,0.017318400740623473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,1,128,1,fp8,fp8,0,0.012964800000190735
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,2,128,1,bfloat16,bfloat16,0,0.012129600346088409
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,2,128,1,bfloat16,fp8,0,0.012595200538635254
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,2,128,1,fp8,fp8,0,0.012699200212955475
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,1,128,1,bfloat16,bfloat16,0,0.01210239976644516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,1,128,1,bfloat16,fp8,0,0.012880000472068786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,2,1,128,1,fp8,fp8,0,0.012724800407886505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,2,128,1,bfloat16,bfloat16,0,0.011849600076675414
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,2,128,1,bfloat16,fp8,0,0.012449599802494049
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,2,128,1,fp8,fp8,0,0.012540799379348756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,1,128,1,bfloat16,bfloat16,0,0.012111999839544297
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,1,128,1,bfloat16,fp8,0,0.012671999633312225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,2,1,128,1,fp8,fp8,0,0.012756800651550293
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,2,128,1,bfloat16,bfloat16,0,0.011796800047159195
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,2,128,1,bfloat16,fp8,0,0.012505599856376648
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,2,128,1,fp8,fp8,0,0.012606400251388549
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,1,128,1,bfloat16,bfloat16,0,0.011982399970293045
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,1,128,1,bfloat16,fp8,0,0.012740799784660339
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,2,1,128,1,fp8,fp8,0,0.012649600207805634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,2,1,128,1,bfloat16,bfloat16,0,0.023228800296783446
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,2,1,128,1,bfloat16,fp8,0,0.024979199469089507
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,32,2,1,128,1,fp8,fp8,0,0.025142401456832886
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,2,128,1,bfloat16,bfloat16,0,0.02011680006980896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,2,128,1,bfloat16,fp8,0,0.021587200462818146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,2,128,1,fp8,fp8,0,0.021348799765110015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,1,128,1,bfloat16,bfloat16,0,0.01603199988603592
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,1,128,1,bfloat16,fp8,0,0.01671839952468872
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,2,1,128,1,fp8,fp8,0,0.01661120057106018
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,2,128,1,bfloat16,bfloat16,0,0.014502400159835815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,2,128,1,bfloat16,fp8,0,0.014880000054836274
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,2,128,1,fp8,fp8,0,0.014959999918937683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,1,128,1,bfloat16,bfloat16,0,0.013787199556827546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,1,128,1,bfloat16,fp8,0,0.014800000190734863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,2,1,128,1,fp8,fp8,0,0.014646400511264802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,2,128,1,bfloat16,bfloat16,0,0.013128000497817992
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,2,128,1,bfloat16,fp8,0,0.013929599523544311
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,2,128,1,fp8,fp8,0,0.01404159963130951
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,1,128,1,bfloat16,bfloat16,0,0.012868799269199371
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,1,128,1,bfloat16,fp8,0,0.013867199420928955
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,2,1,128,1,fp8,fp8,0,0.013950400054454803
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,2,128,1,bfloat16,bfloat16,0,0.012763200700283051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,2,128,1,bfloat16,fp8,0,0.01324159950017929
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,2,128,1,fp8,fp8,0,0.013123199343681335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,1,128,1,bfloat16,bfloat16,0,0.012641599774360657
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,1,128,1,bfloat16,fp8,0,0.013060800731182098
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,2,1,128,1,fp8,fp8,0,0.01297439932823181
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,2,128,1,bfloat16,fp8,0,0.012948800623416901
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,2,128,1,fp8,fp8,0,0.012894399464130402
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,1,128,1,bfloat16,bfloat16,0,0.012121599912643433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,1,128,1,bfloat16,fp8,0,0.01263359934091568
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,1,128,1,fp8,fp8,0,0.012783999741077422
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,2,128,1,bfloat16,bfloat16,0,0.011681599915027619
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,2,128,1,bfloat16,fp8,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,2,128,1,fp8,fp8,0,0.012401600182056428
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,1,128,1,bfloat16,bfloat16,0,0.011910399794578553
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,1,128,1,bfloat16,fp8,0,0.012467200309038163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,2,1,128,1,fp8,fp8,0,0.012734399735927581
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,2,128,1,bfloat16,bfloat16,0,0.011584000289440155
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,2,128,1,bfloat16,fp8,0,0.012563200294971466
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,2,128,1,bfloat16,bfloat16,0,0.013499200344085693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,2,128,1,fp8,fp8,0,0.012305600196123123
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,1,128,1,bfloat16,bfloat16,0,0.01191679984331131
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,1,128,1,fp8,fp8,0,0.012451200187206269
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,2,1,128,1,bfloat16,fp8,0,0.012363199889659882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,2,128,1,bfloat16,bfloat16,0,0.011724799871444702
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,2,128,1,bfloat16,fp8,0,0.012347199767827988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,2,128,1,fp8,fp8,0,0.012366399914026261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,1,128,1,bfloat16,fp8,0,0.012507200241088867
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,1,128,1,fp8,fp8,0,0.012366399914026261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,2,1,128,1,bfloat16,bfloat16,0,0.022363199293613432
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,2,1,128,1,bfloat16,fp8,0,0.023347200453281404
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,2,1,128,1,bfloat16,bfloat16,0,0.012780800461769104
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,16,2,1,128,1,fp8,fp8,0,0.02356639951467514
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,2,128,1,bfloat16,bfloat16,0,0.01927199959754944
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,2,128,1,bfloat16,fp8,0,0.020606400072574617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,2,128,1,fp8,fp8,0,0.02062239944934845
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,1,128,1,bfloat16,bfloat16,0,0.01555359959602356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,1,128,1,bfloat16,fp8,0,0.016228799521923066
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,2,1,128,1,fp8,fp8,0,0.016171200573444365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,2,1,128,1,bfloat16,fp8,0,0.012904000282287598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,2,128,1,bfloat16,fp8,0,0.014761599898338317
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,2,128,1,fp8,fp8,0,0.014646400511264802
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,1,128,1,bfloat16,bfloat16,0,0.01388320028781891
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,1,128,1,bfloat16,fp8,0,0.014555199444293976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,1,128,1,fp8,fp8,0,0.014606399834156037
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,2,128,1,bfloat16,bfloat16,0,0.012899200618267059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,2,128,1,bfloat16,fp8,0,0.013577599823474885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,2,128,1,fp8,fp8,0,0.013540799915790557
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,1,128,1,bfloat16,bfloat16,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,1,128,1,bfloat16,fp8,0,0.01340319961309433
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,2,1,128,1,fp8,fp8,0,0.013646399974822998
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,2,128,1,bfloat16,bfloat16,0,0.012169600278139115
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,2,128,1,bfloat16,fp8,0,0.012761600315570831
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,2,128,1,fp8,fp8,0,0.012971200048923492
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,1,128,1,bfloat16,bfloat16,0,0.012030400335788727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,1,128,1,bfloat16,fp8,0,0.012736000120639801
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,2,1,128,1,fp8,fp8,0,0.012916800379753113
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,2,128,1,bfloat16,fp8,0,0.012603199481964112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,2,128,1,fp8,fp8,0,0.012822400033473968
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,1,128,1,bfloat16,bfloat16,0,0.01188960000872612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,1,128,1,bfloat16,fp8,0,0.012625600397586822
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,1,128,1,fp8,fp8,0,0.012598399817943574
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,2,128,1,bfloat16,bfloat16,0,0.011588799953460693
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,2,128,1,bfloat16,fp8,0,0.012366399914026261
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,2,128,1,fp8,fp8,0,0.01223680004477501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,1,128,1,bfloat16,bfloat16,0,0.011558400094509124
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,1,128,1,bfloat16,fp8,0,0.012331199645996094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,2,1,128,1,fp8,fp8,0,0.012347199767827988
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,2,128,1,bfloat16,bfloat16,0,0.011580800265073776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,2,128,1,bfloat16,fp8,0,0.012300799787044524
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,2,128,1,fp8,fp8,0,0.012281599640846252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,1,128,1,bfloat16,bfloat16,0,0.011451199650764465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,1,128,1,bfloat16,fp8,0,0.012200000137090683
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,2,1,128,1,fp8,fp8,0,0.012167999893426895
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,2,2,128,1,bfloat16,bfloat16,0,0.012379200011491776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,2,128,1,bfloat16,bfloat16,0,0.011484800279140473
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,2,128,1,bfloat16,fp8,0,0.012409599870443344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,2,128,1,fp8,fp8,0,0.01231200024485588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,1,128,1,bfloat16,bfloat16,0,0.011691199988126755
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,1,128,1,bfloat16,fp8,0,0.012307199835777282
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,2,1,128,1,fp8,fp8,0,0.01244639977812767
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,2,1,128,1,bfloat16,bfloat16,0,0.020815999805927278
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,2,1,128,1,bfloat16,fp8,0,0.022729599475860597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,256,1,2,1,128,1,fp8,fp8,0,0.022412799298763275
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,2,2,128,1,bfloat16,bfloat16,0,0.018667200207710268
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,2,2,128,1,bfloat16,fp8,0,0.019838400185108185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,2,2,128,1,fp8,fp8,0,0.020286400616168977
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,2,1,128,1,bfloat16,bfloat16,0,0.014502400159835815
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,2,1,128,1,bfloat16,fp8,0,0.016012799739837647
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,2,1,128,1,fp8,fp8,0,0.015241600573062897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,2,2,128,1,bfloat16,bfloat16,0,0.01345600038766861
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,2,2,128,1,bfloat16,fp8,0,0.014262400567531586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,2,2,128,1,fp8,fp8,0,0.013817599415779114
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,2,1,128,1,bfloat16,bfloat16,0,0.011715199798345566
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,2,1,128,1,bfloat16,bfloat16,0,0.013315199315547943
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,2,1,128,1,bfloat16,fp8,0,0.014113600552082061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,2,1,128,1,fp8,fp8,0,0.014452800154685974
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,2,2,128,1,bfloat16,bfloat16,0,0.012559999525547028
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,2,2,128,1,bfloat16,fp8,0,0.01329759955406189
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,2,2,128,1,fp8,fp8,0,0.013681599497795105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,2,1,128,1,bfloat16,bfloat16,0,0.0123648002743721
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,2,1,128,1,bfloat16,fp8,0,0.013289600610733032
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,2,1,128,1,fp8,fp8,0,0.013715200126171112
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,2,2,128,1,bfloat16,bfloat16,0,0.01191840022802353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,2,2,128,1,bfloat16,fp8,0,0.012918399274349212
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,2,2,128,1,fp8,fp8,0,0.012860800325870513
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,2,1,128,1,bfloat16,bfloat16,0,0.011769600212574005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,2,1,128,1,bfloat16,fp8,0,0.012671999633312225
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,2,1,128,1,fp8,fp8,0,0.01282079964876175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,2,2,128,1,bfloat16,bfloat16,0,0.011793600022792816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,2,2,128,1,bfloat16,bfloat16,0,0.01401280015707016
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,2,2,128,1,bfloat16,fp8,0,0.012585599720478059
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,2,2,128,1,fp8,fp8,0,0.012718400359153748
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,2,1,128,1,bfloat16,bfloat16,0,0.01170239970088005
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,2,1,128,1,bfloat16,fp8,0,0.012408000230789185
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,2,2,128,1,bfloat16,bfloat16,0,0.011767999827861786
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,2,2,128,1,bfloat16,fp8,0,0.012281599640846252
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,2,2,128,1,fp8,fp8,0,0.012254399806261062
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,2,1,128,1,bfloat16,bfloat16,0,0.011764799803495407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,2,1,128,1,bfloat16,fp8,0,0.012151999771595002
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,2,1,128,1,fp8,fp8,0,0.01223680004477501
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,2,2,128,1,bfloat16,bfloat16,0,0.011745599657297134
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,2,2,128,1,bfloat16,fp8,0,0.012132800370454788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,2,2,128,1,fp8,fp8,0,0.012257599830627441
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,2,1,128,1,bfloat16,bfloat16,0,0.011777599900960922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,2,1,128,1,bfloat16,fp8,0,0.012134400010108948
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,2,1,128,1,fp8,fp8,0,0.012195199728012085
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,2,2,128,1,bfloat16,bfloat16,0,0.011614400148391723
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,2,2,128,1,bfloat16,fp8,0,0.012147200107574464
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,2,2,128,1,fp8,fp8,0,0.01226240023970604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,2,1,128,1,bfloat16,bfloat16,0,0.011416000127792359
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,2,1,128,1,fp8,fp8,0,0.01213119998574257
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,1,1,128,1,bfloat16,bfloat16,0,0.4573455810546875
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,1,1,128,1,bfloat16,fp8,0,0.4356256008148193
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16384,1,1,128,1,fp8,fp8,0,0.4362095832824707
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,1,1,128,1,bfloat16,bfloat16,0,0.23346080780029296
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,1,1,128,1,fp8,fp8,0,0.22386720180511474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,1,1,128,1,bfloat16,bfloat16,0,0.2186959981918335
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,1,1,128,1,bfloat16,fp8,0,0.21468958854675294
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16384,1,1,128,1,fp8,fp8,0,0.21460959911346436
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,1,1,128,1,bfloat16,bfloat16,0,0.3213056087493896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,1,1,128,1,bfloat16,fp8,0,0.3111135959625244
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,1,1,128,1,bfloat16,bfloat16,0,0.18048160076141356
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,12288,1,1,128,1,fp8,fp8,0,0.31122560501098634
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,1,1,128,1,bfloat16,fp8,0,0.17414239645004273
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,12288,1,1,128,1,fp8,fp8,0,0.1733680009841919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,1,1,128,1,bfloat16,bfloat16,0,0.10372960567474365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,1,1,128,1,bfloat16,fp8,0,0.10048160552978516
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,12288,1,1,128,1,fp8,fp8,0,0.09971359968185425
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,1,1,128,1,bfloat16,bfloat16,0,0.2539216041564941
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,1,1,128,1,bfloat16,fp8,0,0.2444256067276001
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,10240,1,1,128,1,fp8,fp8,0,0.24394240379333496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,1,1,128,1,bfloat16,bfloat16,0,0.15161440372467042
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,1,1,128,1,bfloat16,fp8,0,0.14672319889068602
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,10240,1,1,128,1,fp8,fp8,0,0.14583359956741332
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,1,1,128,1,bfloat16,bfloat16,0,0.08705919981002808
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,2,1,128,1,fp8,fp8,0,0.012561599910259246
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,1,1,128,1,bfloat16,fp8,0,0.08552320003509521
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,10240,1,1,128,1,fp8,fp8,0,0.08528479933738708
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,1,1,128,1,bfloat16,bfloat16,0,0.3059391975402832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,1,1,128,1,bfloat16,bfloat16,0,0.18424639701843262
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,2,2,128,1,bfloat16,bfloat16,0,0.011852800101041793
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,1,1,128,1,fp8,fp8,0,0.2947391986846924
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,2,1,128,1,bfloat16,fp8,0,0.012406399846076966
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,1,1,128,1,bfloat16,fp8,0,0.17742719650268554
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,1,1,128,1,bfloat16,bfloat16,0,0.11798080205917358
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,1,1,128,1,bfloat16,fp8,0,0.12073279619216919
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,8192,1,1,128,1,fp8,fp8,0,0.1772160053253174
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,1,1,128,1,bfloat16,bfloat16,0,0.07014080286026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,8192,1,1,128,1,fp8,fp8,0,0.11961280107498169
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,1,1,128,1,bfloat16,fp8,0,0.07140319943428039
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,8192,1,1,128,1,fp8,fp8,0,0.07105119824409485
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,1,1,128,1,bfloat16,bfloat16,0,0.11530560255050659
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,1,1,128,1,bfloat16,bfloat16,0,0.19410239458084105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,1,1,128,1,bfloat16,fp8,0,0.1885807991027832
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,6144,1,1,128,1,fp8,fp8,0,0.18889440298080445
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,1,1,128,1,bfloat16,fp8,0,0.11141120195388794
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,6144,1,1,128,1,fp8,fp8,0,0.10962400436401368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,1,1,128,1,bfloat16,fp8,0,0.07179200053215026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,1,1,128,1,fp8,fp8,0,0.07167519927024842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,1,1,128,1,bfloat16,bfloat16,0,0.04393439888954163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,1,1,128,1,bfloat16,fp8,0,0.044516798853874204
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,6144,1,1,128,1,fp8,fp8,0,0.04451839923858643
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,1,1,128,1,bfloat16,bfloat16,0,0.18442239761352539
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,1,1,128,1,bfloat16,fp8,0,0.1792304039001465
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,4096,1,1,128,1,fp8,fp8,0,0.17834880352020263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,1,1,128,1,bfloat16,bfloat16,0,0.11170079708099365
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,1,1,128,1,bfloat16,fp8,0,0.10872160196304322
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,4096,1,1,128,1,fp8,fp8,0,0.10932159423828125
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,1,1,128,1,bfloat16,bfloat16,0,0.06820639967918396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,1,1,128,1,bfloat16,fp8,0,0.0687600016593933
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,4096,1,1,128,1,fp8,fp8,0,0.0691103994846344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,1,1,128,1,bfloat16,bfloat16,0,0.04507359862327576
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,1,1,128,1,bfloat16,fp8,0,0.04583039879798889
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,4096,1,1,128,1,fp8,fp8,0,0.045814400911331175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,1,1,128,1,bfloat16,bfloat16,0,0.02911520004272461
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,1,1,128,1,bfloat16,fp8,0,0.030830401182174682
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,4096,1,1,128,1,fp8,fp8,0,0.030612799525260925
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,2,2,128,1,fp8,fp8,0,0.014319999516010285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,1,1,128,1,bfloat16,bfloat16,0,0.12236160039901733
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,1,1,128,1,bfloat16,fp8,0,0.1206096053123474
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,1,1,128,1,bfloat16,bfloat16,0,0.07528799772262573
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,1,1,128,1,bfloat16,fp8,0,0.07442079782485962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,3072,1,1,128,1,fp8,fp8,0,0.07618880271911621
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,1,1,128,1,bfloat16,bfloat16,0,0.054094398021698
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,1,1,128,1,bfloat16,fp8,0,0.052027201652526854
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,1,1,128,1,bfloat16,bfloat16,0,0.032419198751449586
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,3072,1,1,128,1,fp8,fp8,0,0.05332319736480713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,1,1,128,1,bfloat16,fp8,0,0.0322735995054245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,3072,1,1,128,1,fp8,fp8,0,0.033076798915863036
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,1,1,128,1,bfloat16,bfloat16,0,0.02460159957408905
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,1,1,128,1,bfloat16,fp8,0,0.023800000548362732
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,3072,1,1,128,1,fp8,fp8,0,0.024527999758720397
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,1,1,128,1,bfloat16,bfloat16,0,0.1225119948387146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,1,1,128,1,bfloat16,fp8,0,0.11668640375137329
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,2048,1,1,128,1,fp8,fp8,0,0.11839040517807006
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,1,1,128,1,bfloat16,bfloat16,0,0.07411519885063171
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,8192,1,1,128,1,bfloat16,fp8,0,0.291865611076355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,1,1,128,1,bfloat16,fp8,0,0.07001439929008484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,2048,1,1,128,1,fp8,fp8,0,0.07229440212249756
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,1,1,128,1,bfloat16,bfloat16,0,0.044119998812675476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,1,1,128,1,bfloat16,fp8,0,0.04351199865341186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,2048,1,1,128,1,fp8,fp8,0,0.0428191989660263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,1,1,128,1,bfloat16,bfloat16,0,0.03357760012149811
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,1,1,128,1,bfloat16,fp8,0,0.03409439921379089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,2048,1,1,128,1,fp8,fp8,0,0.03401280045509338
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,1,1,128,1,bfloat16,bfloat16,0,0.023444800078868865
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,1,1,128,1,bfloat16,fp8,0,0.025145599246025087
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,2048,1,1,128,1,fp8,fp8,0,0.025012800097465517
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,1,1,128,1,bfloat16,bfloat16,0,0.020427200198173522
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,1,1,128,1,bfloat16,fp8,0,0.019702400267124175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,2048,1,1,128,1,fp8,fp8,0,0.020315200090408325
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,1,1,128,1,bfloat16,bfloat16,0,0.08535360097885132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,1,1,128,1,bfloat16,bfloat16,0,0.05309919714927673
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,1,1,128,1,bfloat16,fp8,0,0.0839136004447937
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1536,1,1,128,1,fp8,fp8,0,0.08254879713058472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16384,1,1,128,1,bfloat16,fp8,0,0.22260799407958984
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,1,1,128,1,bfloat16,fp8,0,0.05213599801063538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1536,1,1,128,1,fp8,fp8,0,0.051806402206420896
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,1,1,128,1,bfloat16,bfloat16,0,0.041860800981521604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,1,1,128,1,bfloat16,fp8,0,0.041529598832130435
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,6144,1,1,128,1,bfloat16,bfloat16,0,0.0718608021736145
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1536,1,1,128,1,fp8,fp8,0,0.042268800735473636
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,1,1,128,1,bfloat16,bfloat16,0,0.026902401447296144
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,1,1,128,1,bfloat16,fp8,0,0.027504000067710876
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1536,1,1,128,1,fp8,fp8,0,0.027403199672698976
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,1,1,128,1,bfloat16,fp8,0,0.020576000213623047
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,1,1,128,1,fp8,fp8,0,0.020545600354671477
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,1,1,128,1,bfloat16,fp8,0,0.02009280025959015
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,1,1,128,1,fp8,fp8,0,0.02011200040578842
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,1,1,128,1,bfloat16,bfloat16,0,0.08938559889793396
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,1,1,128,1,bfloat16,bfloat16,0,0.05396639704704285
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,1,1,128,1,bfloat16,fp8,0,0.09039199948310853
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1024,1,1,128,1,fp8,fp8,0,0.09194719791412354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,1,1,128,1,bfloat16,bfloat16,0,0.031307199597358705
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,1,1,128,1,fp8,fp8,0,0.05393760204315186
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,1,1,128,1,bfloat16,fp8,0,0.03240160048007965
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1024,1,1,128,1,fp8,fp8,0,0.032179200649261476
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,1,1,128,1,bfloat16,fp8,0,0.028835201263427736
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,1,1,128,1,fp8,fp8,0,0.028513601422309874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,1,1,128,1,bfloat16,bfloat16,0,0.021731199324131013
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,1,1,128,1,bfloat16,fp8,0,0.021371200680732727
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1024,1,1,128,1,fp8,fp8,0,0.021393600106239318
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,1,1,128,1,bfloat16,bfloat16,0,0.018408000469207764
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,3072,1,1,128,1,fp8,fp8,0,0.12019200325012207
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,1,1,128,1,bfloat16,fp8,0,0.019547200202941893
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1024,1,1,128,1,fp8,fp8,0,0.019891199469566346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,1,1,128,1,bfloat16,bfloat16,0,0.017931200563907623
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,1,1,128,1,bfloat16,fp8,0,0.018964800238609313
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1024,1,1,128,1,fp8,fp8,0,0.01887039989233017
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,1,1,128,1,bfloat16,bfloat16,0,0.06972960233688355
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,1,1,128,1,bfloat16,fp8,0,0.06906399726867676
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,1,1,128,1,bfloat16,bfloat16,0,0.04033760130405426
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,512,1,1,128,1,fp8,fp8,0,0.0701632022857666
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,1,1,128,1,bfloat16,fp8,0,0.04014880061149597
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,512,1,1,128,1,fp8,fp8,0,0.04076479971408844
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,1,1,128,1,bfloat16,bfloat16,0,0.023054400086402894
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,1,1,128,1,bfloat16,fp8,0,0.02465279996395111
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,512,1,1,128,1,fp8,fp8,0,0.024348799884319306
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,1,1,128,1,bfloat16,fp8,0,0.021116800606250763
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,1,1,128,1,fp8,fp8,0,0.021096000075340272
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,1,1,128,1,bfloat16,bfloat16,0,0.018296000361442567
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,1,1,128,1,bfloat16,fp8,0,0.019963200390338897
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,512,1,1,128,1,fp8,fp8,0,0.020001600682735442
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,1,1,128,1,bfloat16,bfloat16,0,0.017097599804401398
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,1,1,128,1,bfloat16,fp8,0,0.018987199664115904
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,512,1,1,128,1,fp8,fp8,0,0.018982400000095368
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,1,1,128,1,bfloat16,bfloat16,0,0.016804799437522888
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,1,1,128,1,bfloat16,fp8,0,0.018699200451374055
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,1,1,128,1,bfloat16,bfloat16,0,0.016571199893951415
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1536,1,1,128,1,bfloat16,bfloat16,0,0.020436799526214598
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1536,1,1,128,1,bfloat16,bfloat16,0,0.018702399730682374
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,1,1,128,1,bfloat16,fp8,0,0.018910400569438934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,512,1,1,128,1,fp8,fp8,0,0.018859200179576874
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,1,1,128,1,bfloat16,fp8,0,0.0349375993013382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,1,1,128,1,fp8,fp8,0,0.03503359854221344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,1,1,128,1,bfloat16,bfloat16,0,0.020721599459648132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,1,1,128,1,bfloat16,fp8,0,0.021396799385547637
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,256,1,1,128,1,fp8,fp8,0,0.02136960029602051
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,1,1,128,1,bfloat16,bfloat16,0,0.01727679967880249
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,1,1,128,1,bfloat16,fp8,0,0.018136000633239745
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1024,1,1,128,1,bfloat16,fp8,0,0.05338559746742248
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,1,1,128,1,bfloat16,bfloat16,0,0.01571040004491806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,1,1,128,1,bfloat16,fp8,0,0.01658879965543747
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1024,1,1,128,1,bfloat16,bfloat16,0,0.027059200406074523
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,1,1,128,1,bfloat16,bfloat16,0,0.014271999895572662
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,1,1,128,1,bfloat16,fp8,0,0.01515199989080429
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,256,1,1,128,1,fp8,fp8,0,0.015479999780654907
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,1,1,128,1,bfloat16,bfloat16,0,0.013894400000572205
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,1,1,128,1,bfloat16,fp8,0,0.014931200444698334
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,256,1,1,128,1,fp8,fp8,0,0.014985600113868713
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,1,1,128,1,bfloat16,bfloat16,0,0.013737599551677703
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,1,1,128,1,bfloat16,fp8,0,0.014948800206184387
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,256,1,1,128,1,fp8,fp8,0,0.014899200201034546
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,1,1,128,1,bfloat16,bfloat16,0,0.013598400354385375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,1,1,128,1,bfloat16,fp8,0,0.014803199470043183
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,256,1,1,128,1,fp8,fp8,0,0.014689600467681885
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,1,1,128,1,bfloat16,bfloat16,0,0.020904000103473663
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,1,1,128,1,bfloat16,fp8,0,0.020494399964809416
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,128,1,1,128,1,fp8,fp8,0,0.020657600462436677
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,1,1,128,1,bfloat16,bfloat16,0,0.016923199594020843
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,1,1,128,1,bfloat16,fp8,0,0.016892799735069276
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,128,1,1,128,1,fp8,fp8,0,0.016897599399089813
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,1,1,128,1,bfloat16,bfloat16,0,0.014348800480365752
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,1,1,128,1,bfloat16,fp8,0,0.014912000298500061
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,128,1,1,128,1,fp8,fp8,0,0.014878399670124054
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,1,1,128,1,bfloat16,bfloat16,0,0.01300159990787506
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,1,1,128,1,bfloat16,fp8,0,0.013395200669765472
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,128,1,1,128,1,fp8,fp8,0,0.013387200236320496
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,1,1,128,1,bfloat16,bfloat16,0,0.012436799705028534
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,1,1,128,1,fp8,fp8,0,0.012956799566745758
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,512,1,1,128,1,bfloat16,bfloat16,0,0.01972319930791855
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,1,1,128,1,bfloat16,bfloat16,0,0.012228800356388092
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,1,1,128,1,bfloat16,fp8,0,0.012750400602817536
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,1,1,128,1,bfloat16,bfloat16,0,0.012049599736928939
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,1,1,128,1,bfloat16,fp8,0,0.01271039992570877
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,128,1,1,128,1,fp8,fp8,0,0.012678399682044983
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,1,1,128,1,bfloat16,bfloat16,0,0.01191840022802353
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,1,1,128,1,bfloat16,fp8,0,0.012611199915409089
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,128,1,1,128,1,fp8,fp8,0,0.012612800300121307
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,1,1,128,1,bfloat16,bfloat16,0,0.01752000004053116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,1,1,128,1,bfloat16,fp8,0,0.018182399868965148
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,64,1,1,128,1,fp8,fp8,0,0.018198400735855103
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,1,1,128,1,bfloat16,bfloat16,0,0.014683200418949128
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,512,1,1,128,1,fp8,fp8,0,0.018990400433540344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,1,1,128,1,bfloat16,fp8,0,0.01541920006275177
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,1,1,128,1,bfloat16,bfloat16,0,0.013177600502967835
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,64,1,1,128,1,fp8,fp8,0,0.015353600680828094
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,256,1,1,128,1,bfloat16,bfloat16,0,0.03613280057907105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,1,1,128,1,bfloat16,fp8,0,0.013689599931240082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,64,1,1,128,1,fp8,fp8,0,0.013684800267219544
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,1,1,128,1,bfloat16,bfloat16,0,0.012297599762678146
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,1,1,128,1,fp8,fp8,0,0.01287200003862381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,1,1,128,1,bfloat16,bfloat16,0,0.012054400146007537
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,1,1,128,1,bfloat16,fp8,0,0.012982399761676788
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,64,1,1,128,1,fp8,fp8,0,0.012759999930858612
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,1,1,128,1,bfloat16,bfloat16,0,0.011777599900960922
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,1,1,128,1,bfloat16,fp8,0,0.012593600153923034
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,64,1,1,128,1,fp8,fp8,0,0.012425599992275238
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,1,1,128,1,bfloat16,bfloat16,0,0.011793600022792816
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,1,1,128,1,bfloat16,fp8,0,0.012596799433231354
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,64,1,1,128,1,fp8,fp8,0,0.012399999797344208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,1,1,128,1,bfloat16,bfloat16,0,0.011760000139474869
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,256,1,1,128,1,fp8,fp8,0,0.016420799493789672
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,1,1,128,1,bfloat16,fp8,0,0.012404800206422806
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,1,1,128,1,bfloat16,bfloat16,0,0.01594240069389343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,1,1,128,1,bfloat16,fp8,0,0.016942399740219116
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,32,1,1,128,1,fp8,fp8,0,0.016832000017166136
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,1,1,128,1,bfloat16,bfloat16,0,0.01419840008020401
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,1,1,128,1,bfloat16,fp8,0,0.014591999351978302
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,32,1,1,128,1,fp8,fp8,0,0.014435200393199921
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,1,1,128,1,bfloat16,bfloat16,0,0.012788799405097962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,1,1,128,1,bfloat16,fp8,0,0.01356160044670105
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,32,1,1,128,1,fp8,fp8,0,0.013449600338935852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,1,1,128,1,bfloat16,bfloat16,0,0.01215839982032776
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,1,1,128,1,bfloat16,fp8,0,0.012724800407886505
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,32,1,1,128,1,fp8,fp8,0,0.012852799892425538
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,1,1,128,1,bfloat16,bfloat16,0,0.011854399740695954
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,1,1,128,1,bfloat16,fp8,0,0.012564800679683685
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,32,1,1,128,1,fp8,fp8,0,0.012457600235939026
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,1,1,128,1,bfloat16,bfloat16,0,0.011598400026559829
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,1,1,128,1,bfloat16,fp8,0,0.01218079999089241
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,32,1,1,128,1,fp8,fp8,0,0.012328000366687774
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,1,1,128,1,bfloat16,bfloat16,0,0.011406400054693223
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,1,1,128,1,bfloat16,fp8,0,0.01231200024485588
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,32,1,1,128,1,fp8,fp8,0,0.012204799801111221
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,1,1,128,1,bfloat16,bfloat16,0,0.011603199690580369
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,1,1,128,1,bfloat16,fp8,0,0.012225600332021714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,32,1,1,128,1,fp8,fp8,0,0.012462399899959564
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,1,1,128,1,bfloat16,bfloat16,0,0.015600000321865082
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,1,1,128,1,bfloat16,fp8,0,0.016289600729942323
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,128,1,1,128,1,bfloat16,fp8,0,0.013102400302886962
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,16,1,1,128,1,fp8,fp8,0,0.01613440066576004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,1,1,128,1,bfloat16,bfloat16,0,0.013612799346446991
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,1,1,128,1,bfloat16,fp8,0,0.014099200069904328
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,1,1,128,1,bfloat16,bfloat16,0,0.012772800028324127
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,1,1,128,1,bfloat16,fp8,0,0.01324480026960373
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,16,1,1,128,1,fp8,fp8,0,0.013449600338935852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,128,1,1,128,1,fp8,fp8,0,0.012798400223255157
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,1,1,128,1,bfloat16,bfloat16,0,0.011823999881744384
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,1,1,128,1,bfloat16,fp8,0,0.012774400413036346
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,1,1,128,1,bfloat16,bfloat16,0,0.011713600158691407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,1,1,128,1,bfloat16,fp8,0,0.012428800016641617
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,16,1,1,128,1,fp8,fp8,0,0.012484800070524216
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,1,1,128,1,bfloat16,bfloat16,0,0.011564800143241882
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,1,1,128,1,bfloat16,fp8,0,0.012329600006341934
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,16,1,1,128,1,fp8,fp8,0,0.01223519966006279
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,1,1,128,1,bfloat16,bfloat16,0,0.011539199948310852
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,1,1,128,1,bfloat16,fp8,0,0.012276799976825714
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,16,1,1,128,1,fp8,fp8,0,0.012358400225639343
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,1,1,128,1,bfloat16,bfloat16,0,0.011582399904727935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,1,1,128,1,bfloat16,fp8,0,0.012296000123023986
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,16,1,1,128,1,fp8,fp8,0,0.012380799651145935
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,1,1,128,1,bfloat16,bfloat16,0,0.014745600521564484
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,1,1,128,1,bfloat16,fp8,0,0.015936000645160674
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,128,1,1,1,128,1,fp8,fp8,0,0.01560479998588562
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,1,1,128,1,bfloat16,bfloat16,0,0.013513599336147309
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,1,1,128,1,bfloat16,fp8,0,0.014241600036621093
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,1,1,1,128,1,fp8,fp8,0,0.014238399267196656
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,1,1,128,1,bfloat16,bfloat16,0,0.012371200323104858
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,1,1,128,1,bfloat16,fp8,0,0.013318400084972381
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,1,1,1,128,1,fp8,fp8,0,0.01316480040550232
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,1,1,128,1,bfloat16,bfloat16,0,0.011764799803495407
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,1,1,128,1,bfloat16,fp8,0,0.012435200065374375
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,1,1,1,128,1,fp8,fp8,0,0.012518399953842163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,1,1,128,1,bfloat16,bfloat16,0,0.011611200124025344
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,1,1,128,1,bfloat16,fp8,0,0.012399999797344208
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,8,1,1,1,128,1,fp8,fp8,0,0.012441600114107132
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,1,1,128,1,bfloat16,bfloat16,0,0.011494400352239609
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,64,1,1,128,1,bfloat16,fp8,0,0.01282079964876175
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,1,1,128,1,bfloat16,fp8,0,0.01223360002040863
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,4,1,1,1,128,1,fp8,fp8,0,0.01231359988451004
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,1,1,128,1,bfloat16,bfloat16,0,0.011348800361156463
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,1,1,128,1,bfloat16,fp8,0,0.012100800126791
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,2,1,1,1,128,1,fp8,fp8,0,0.01226079985499382
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,1,1,128,1,bfloat16,bfloat16,0,0.011376000195741653
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,32,256,1,1,128,1,fp8,fp8,0,0.018352000415325163
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,1,1,128,1,fp8,fp8,0,0.01228479966521263
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,64,1,1,128,1,fp8,fp8,0,0.012539200484752655
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,64,16,1,1,128,1,fp8,fp8,0,0.014158399403095245
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,16,16,1,1,128,1,fp8,fp8,0,0.012726399302482604
SGLang,0.5.9,NVIDIA H100 80GB HBM3,context_attention,flash_attention,1,1,1,1,128,1,bfloat16,fp8,0,0.012134400010108948
