framework,version,device,op_name,kernel_source,batch_size,isl,num_heads,num_key_value_heads,head_dim,beam_width,attn_dtype,kv_cache_dtype,step,latency
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,96,1,128,1,float16,float16,0,125.28067626953126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,96,2,128,1,float16,float16,0,126.32396240234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,96,4,128,1,float16,float16,0,126.78072509765624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,96,96,128,1,float16,float16,0,131.04385986328126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,96,96,128,1,float16,float16,0,66.6515625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,96,1,128,1,float16,float16,0,63.44700927734375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,96,2,128,1,float16,float16,0,62.60640258789063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,96,1,128,1,float16,float16,0,255.9880615234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,96,8,128,1,float16,float16,0,126.1838623046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,96,2,128,1,float16,float16,0,254.4009521484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,96,96,128,1,float16,float16,0,33.270013427734376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,96,4,128,1,float16,float16,0,250.261474609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,96,8,128,1,float16,float16,0,252.7067626953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,96,4,128,1,float16,float16,0,63.396453857421875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,96,1,128,1,float16,float16,0,32.1830322265625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,96,2,128,1,float16,float16,0,31.32489929199219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,96,4,128,1,float16,float16,0,32.006100463867185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,96,8,128,1,float16,float16,0,63.88248291015625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,96,8,128,1,float16,float16,0,31.642694091796876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,96,96,128,1,float16,float16,0,75.33751220703125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,96,1,128,1,float16,float16,0,72.1583984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,96,2,128,1,float16,float16,0,72.0099365234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,96,4,128,1,float16,float16,0,71.08235473632813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,96,1,128,1,float16,float16,0,36.09656982421875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,96,2,128,1,float16,float16,0,35.57870788574219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,96,96,128,1,float16,float16,0,38.369781494140625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,96,4,128,1,float16,float16,0,143.71424560546876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,96,1,128,1,float16,float16,0,141.8443115234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,96,2,128,1,float16,float16,0,143.3383056640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,96,8,128,1,float16,float16,0,146.00042724609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,96,96,128,1,float16,float16,0,19.295664978027343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,96,8,128,1,float16,float16,0,71.50840454101562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,96,2,128,1,float16,float16,0,17.708206176757812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,96,1,128,1,float16,float16,0,17.952798461914064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,96,4,128,1,float16,float16,0,18.073344421386718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,96,8,128,1,float16,float16,0,35.680328369140625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,96,4,128,1,float16,float16,0,36.172784423828126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,96,8,128,1,float16,float16,0,18.3888427734375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,96,2,128,1,float16,float16,0,50.36299438476563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,96,96,128,1,float16,float16,0,53.44248046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,96,1,128,1,float16,float16,0,49.64308471679688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,96,4,128,1,float16,float16,0,50.99375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,96,1,128,1,float16,float16,0,24.854840087890626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,96,96,128,1,float16,float16,0,26.86430969238281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,96,1,128,1,float16,float16,0,100.63326416015624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,96,2,128,1,float16,float16,0,25.33056640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,96,2,128,1,float16,float16,0,99.27286987304687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,96,4,128,1,float16,float16,0,99.24721069335938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,96,96,128,1,float16,float16,0,13.672900390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,96,1,128,1,float16,float16,0,12.960420227050781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,96,8,128,1,float16,float16,0,100.5551025390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,96,8,128,1,float16,float16,0,50.58088989257813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,96,2,128,1,float16,float16,0,12.56451187133789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,96,4,128,1,float16,float16,0,24.929437255859376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,96,4,128,1,float16,float16,0,12.502028656005859
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,96,8,128,1,float16,float16,0,25.342962646484374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,96,8,128,1,float16,float16,0,12.742842864990234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,96,1,128,1,float16,float16,0,63.84801025390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,96,2,128,1,float16,float16,0,64.57076416015624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,96,96,128,1,float16,float16,0,68.77261962890626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,96,4,128,1,float16,float16,0,65.22529907226563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,96,96,128,1,float16,float16,0,34.962738037109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,96,1,128,1,float16,float16,0,31.863983154296875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,96,2,128,1,float16,float16,0,32.53162536621094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,96,2,128,1,float16,float16,0,128.838623046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,96,1,128,1,float16,float16,0,130.714697265625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,96,96,128,1,float16,float16,0,17.63813934326172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,96,4,128,1,float16,float16,0,129.819189453125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,96,8,128,1,float16,float16,0,128.43355712890624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,96,8,128,1,float16,float16,0,64.10361328125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,96,4,128,1,float16,float16,0,32.50652160644531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,96,8,128,1,float16,float16,0,32.04178161621094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,96,96,128,1,float16,float16,0,8.801531219482422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,96,1,128,1,float16,float16,0,7.977995300292969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,96,1,128,1,float16,float16,0,16.16761169433594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,96,4,128,1,float16,float16,0,8.090408325195312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,96,2,128,1,float16,float16,0,8.134774780273437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,96,2,128,1,float16,float16,0,16.433370971679686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,96,4,128,1,float16,float16,0,16.353678894042968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,96,8,128,1,float16,float16,0,16.42029266357422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,96,8,128,1,float16,float16,0,8.134114837646484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,96,1,128,1,float16,float16,0,36.92046203613281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,96,2,128,1,float16,float16,0,36.886660766601565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,96,4,128,1,float16,float16,0,36.45686645507813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,96,96,128,1,float16,float16,0,40.85895690917969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,96,1,128,1,float16,float16,0,18.173670959472656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,96,96,128,1,float16,float16,0,20.362159729003906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,96,2,128,1,float16,float16,0,18.562887573242186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,96,1,128,1,float16,float16,0,72.50144653320312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,96,4,128,1,float16,float16,0,72.6497314453125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,96,2,128,1,float16,float16,0,73.35734252929687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,96,8,128,1,float16,float16,0,73.82669677734376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,96,8,128,1,float16,float16,0,37.17973937988281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,96,96,128,1,float16,float16,0,10.257695770263672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,96,4,128,1,float16,float16,0,18.246499633789064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,96,1,128,1,float16,float16,0,4.659801483154297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,96,96,128,1,float16,float16,0,5.131063842773438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,96,1,128,1,float16,float16,0,9.062032318115234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,96,2,128,1,float16,float16,0,9.170761871337891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,96,8,128,1,float16,float16,0,18.698109436035157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,96,4,128,1,float16,float16,0,9.254798126220702
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,96,8,128,1,float16,float16,0,9.34912338256836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,96,2,128,1,float16,float16,0,4.609708786010742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,96,4,128,1,float16,float16,0,4.713471984863281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,96,8,128,1,float16,float16,0,4.70446891784668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,96,2,128,1,float16,float16,0,33.44895935058594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,96,1,128,1,float16,float16,0,33.68318176269531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,96,96,128,1,float16,float16,0,37.572637939453124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,96,4,128,1,float16,float16,0,33.20139770507812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,96,96,128,1,float16,float16,0,19.2202392578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,96,1,128,1,float16,float16,0,16.477383422851563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,96,2,128,1,float16,float16,0,16.85437927246094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,96,1,128,1,float16,float16,0,65.64547119140624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,96,2,128,1,float16,float16,0,65.6451416015625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,96,4,128,1,float16,float16,0,66.62416381835938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,96,8,128,1,float16,float16,0,67.22811889648438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,96,8,128,1,float16,float16,0,33.71666564941406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,96,96,128,1,float16,float16,0,9.513116455078125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,96,1,128,1,float16,float16,0,8.238295745849609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,96,4,128,1,float16,float16,0,16.917449951171875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,96,2,128,1,float16,float16,0,8.246201324462891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,96,8,128,1,float16,float16,0,16.662725830078124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,96,4,128,1,float16,float16,0,8.400057220458985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,96,96,128,1,float16,float16,0,4.8461151123046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,96,1,128,1,float16,float16,0,4.142825698852539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,96,96,128,1,float16,float16,0,2.410833549499512
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,96,2,128,1,float16,float16,0,4.106167984008789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,96,8,128,1,float16,float16,0,8.406864166259766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,96,1,128,1,float16,float16,0,2.0709375381469726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,96,2,128,1,float16,float16,0,2.1375776290893556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,96,4,128,1,float16,float16,0,4.212220764160156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,96,8,128,1,float16,float16,0,4.199337768554687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,96,4,128,1,float16,float16,0,2.135207939147949
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,96,8,128,1,float16,float16,0,2.135107231140137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,96,1,128,1,float16,float16,0,18.76128234863281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,96,2,128,1,float16,float16,0,19.174349975585937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,96,4,128,1,float16,float16,0,18.945323181152343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,96,96,128,1,float16,float16,0,22.75079345703125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,96,1,128,1,float16,float16,0,9.446121978759766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,96,96,128,1,float16,float16,0,11.35156478881836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,96,2,128,1,float16,float16,0,9.668033599853516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,96,1,128,1,float16,float16,0,37.56221923828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,96,2,128,1,float16,float16,0,38.268133544921874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,96,4,128,1,float16,float16,0,38.34726257324219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,96,8,128,1,float16,float16,0,19.09576873779297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,96,8,128,1,float16,float16,0,38.73171691894531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,96,96,128,1,float16,float16,0,5.683185577392578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,96,4,128,1,float16,float16,0,9.53060302734375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,96,1,128,1,float16,float16,0,2.4161504745483398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,96,1,128,1,float16,float16,0,4.702787017822265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,96,96,128,1,float16,float16,0,2.872257614135742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,96,8,128,1,float16,float16,0,9.697491455078126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,96,2,128,1,float16,float16,0,2.3903583526611327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,96,2,128,1,float16,float16,0,4.752942276000977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,96,4,128,1,float16,float16,0,4.761969757080078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,96,8,128,1,float16,float16,0,4.77167854309082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,96,1,128,1,float16,float16,0,1.2018848419189454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,96,96,128,1,float16,float16,0,1.4767536163330077
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,96,2,128,1,float16,float16,0,1.2077152252197265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,96,8,128,1,float16,float16,0,2.4051488876342773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,96,4,128,1,float16,float16,0,2.418804740905762
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,96,8,128,1,float16,float16,0,1.252783966064453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,96,4,128,1,float16,float16,0,1.2502927780151367
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,96,1,128,1,float16,float16,0,17.77137908935547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,96,2,128,1,float16,float16,0,17.434580993652343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,96,4,128,1,float16,float16,0,17.723069763183595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,96,96,128,1,float16,float16,0,21.75
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,96,96,128,1,float16,float16,0,11.061479949951172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,96,1,128,1,float16,float16,0,8.768006134033204
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,96,1,128,1,float16,float16,0,34.707882690429685
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,96,2,128,1,float16,float16,0,35.278936767578124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,96,8,128,1,float16,float16,0,35.382510375976565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,96,4,128,1,float16,float16,0,35.592901611328124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,96,2,128,1,float16,float16,0,8.70203857421875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,96,8,128,1,float16,float16,0,18.043289184570312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,96,4,128,1,float16,float16,0,8.904952239990234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,96,96,128,1,float16,float16,0,2.8130416870117188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,96,1,128,1,float16,float16,0,4.361848068237305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,96,96,128,1,float16,float16,0,5.483857727050781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,96,2,128,1,float16,float16,0,4.304851150512695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,96,4,128,1,float16,float16,0,4.422417449951172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,96,8,128,1,float16,float16,0,4.386161422729492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,96,8,128,1,float16,float16,0,8.947679901123047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,96,1,128,1,float16,float16,0,2.184931182861328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,96,1,128,1,float16,float16,0,1.11038236618042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,96,2,128,1,float16,float16,0,1.1167072296142577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,96,96,128,1,float16,float16,0,1.4112192153930665
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,96,2,128,1,float16,float16,0,2.204523277282715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,96,4,128,1,float16,float16,0,2.2173999786376952
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,96,1,128,1,float16,float16,0,0.5671199798583985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,96,8,128,1,float16,float16,0,2.2063056945800783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,96,2,128,1,float16,float16,0,0.5860640048980713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,96,4,128,1,float16,float16,0,1.140392017364502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,96,96,128,1,float16,float16,0,0.7456448078155518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,96,8,128,1,float16,float16,0,1.1633824348449706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,96,4,128,1,float16,float16,0,0.6118224143981934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,96,8,128,1,float16,float16,0,0.628385591506958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,96,1,128,1,float16,float16,0,10.193385314941406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,96,2,128,1,float16,float16,0,10.389470672607422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,96,4,128,1,float16,float16,0,10.370559692382812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,96,96,128,1,float16,float16,0,13.648690795898437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,96,96,128,1,float16,float16,0,6.806511688232422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,96,1,128,1,float16,float16,0,5.096212768554688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,96,1,128,1,float16,float16,0,20.207794189453125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,96,2,128,1,float16,float16,0,20.101634216308593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,96,4,128,1,float16,float16,0,20.39248046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,96,8,128,1,float16,float16,0,10.340465545654297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,96,8,128,1,float16,float16,0,20.935690307617186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,96,2,128,1,float16,float16,0,5.156817626953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,96,1,128,1,float16,float16,0,2.4792448043823243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,96,2,128,1,float16,float16,0,2.4568191528320313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,96,4,128,1,float16,float16,0,2.5329856872558594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,96,8,128,1,float16,float16,0,2.578268814086914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,96,96,128,1,float16,float16,0,3.3559600830078127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,96,4,128,1,float16,float16,0,5.1543327331542965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,96,1,128,1,float16,float16,0,1.2536784172058106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,96,2,128,1,float16,float16,0,1.2928720474243165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,96,96,128,1,float16,float16,0,1.7248016357421876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,96,96,128,1,float16,float16,0,0.9019536018371582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,96,4,128,1,float16,float16,0,1.3296079635620117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,96,8,128,1,float16,float16,0,1.327943992614746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,96,8,128,1,float16,float16,0,5.120246505737304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,96,1,128,1,float16,float16,0,0.6652192115783692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,96,2,128,1,float16,float16,0,0.6658656120300293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,96,96,128,1,float16,float16,0,0.48172798156738283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,96,4,128,1,float16,float16,0,0.6847439765930176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,96,1,128,1,float16,float16,0,0.3599711894989014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,96,8,128,1,float16,float16,0,0.6926559925079345
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,96,4,128,1,float16,float16,0,0.3730272054672241
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,96,8,128,1,float16,float16,0,0.3725728034973145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,96,2,128,1,float16,float16,0,0.37269599437713624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,96,1,128,1,float16,float16,0,9.566553497314453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,96,2,128,1,float16,float16,0,9.76928482055664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,96,4,128,1,float16,float16,0,9.83385772705078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,96,96,128,1,float16,float16,0,14.215017700195313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,96,1,128,1,float16,float16,0,4.798844909667968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,96,96,128,1,float16,float16,0,7.08233413696289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,96,2,128,1,float16,float16,0,19.123634338378906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,96,1,128,1,float16,float16,0,19.190182495117188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,96,4,128,1,float16,float16,0,19.755410766601564
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,96,2,128,1,float16,float16,0,4.8939361572265625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,96,8,128,1,float16,float16,0,19.86914520263672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,96,8,128,1,float16,float16,0,9.816680145263671
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,96,4,128,1,float16,float16,0,4.866451263427734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,96,1,128,1,float16,float16,0,2.356817626953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,96,1,128,1,float16,float16,0,1.2056063652038573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,96,96,128,1,float16,float16,0,1.7657712936401366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,96,2,128,1,float16,float16,0,2.39694881439209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,96,8,128,1,float16,float16,0,4.958798217773437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,96,96,128,1,float16,float16,0,3.5030689239501953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,96,4,128,1,float16,float16,0,2.4299295425415037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,96,8,128,1,float16,float16,0,2.4267215728759766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,96,2,128,1,float16,float16,0,1.2052656173706056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,96,4,128,1,float16,float16,0,1.2043408393859862
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,96,96,128,1,float16,float16,0,0.921993637084961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,96,8,128,1,float16,float16,0,1.2119248390197754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,96,4,128,1,float16,float16,0,0.6461728096008301
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,96,2,128,1,float16,float16,0,0.6229231834411622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,96,1,128,1,float16,float16,0,0.33194880485534667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,96,96,128,1,float16,float16,0,0.48665761947631836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,96,8,128,1,float16,float16,0,0.650270414352417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,96,1,128,1,float16,float16,0,0.6246352195739746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,96,4,128,1,float16,float16,0,0.3423824071884155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,96,2,128,1,float16,float16,0,0.33823840618133544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,96,8,128,1,float16,float16,0,0.34747040271759033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,96,1,128,1,float16,float16,0,0.183513605594635
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,96,96,128,1,float16,float16,0,0.2642080068588257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,96,2,128,1,float16,float16,0,0.18239680528640748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,96,8,128,1,float16,float16,0,0.1981536030769348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,96,4,128,1,float16,float16,0,0.19230719804763793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,96,1,128,1,float16,float16,0,5.689424133300781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,96,2,128,1,float16,float16,0,5.799121475219726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,96,4,128,1,float16,float16,0,5.8305919647216795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,96,1,128,1,float16,float16,0,2.8477840423583984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,96,96,128,1,float16,float16,0,10.050991821289063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,96,1,128,1,float16,float16,0,11.362509155273438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,96,96,128,1,float16,float16,0,5.136103820800781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,96,2,128,1,float16,float16,0,11.68988494873047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,96,4,128,1,float16,float16,0,11.905235290527344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,96,8,128,1,float16,float16,0,5.938286590576172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,96,2,128,1,float16,float16,0,2.859511947631836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,96,8,128,1,float16,float16,0,12.14203338623047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,96,4,128,1,float16,float16,0,2.8450719833374025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,96,1,128,1,float16,float16,0,1.407688045501709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,96,8,128,1,float16,float16,0,1.474505615234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,96,2,128,1,float16,float16,0,1.4136128425598145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,96,96,128,1,float16,float16,0,1.3141008377075196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,96,4,128,1,float16,float16,0,1.4318783760070801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,96,8,128,1,float16,float16,0,2.9098783493041993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,96,96,128,1,float16,float16,0,2.579955291748047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,96,1,128,1,float16,float16,0,0.7251327991485595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,96,2,128,1,float16,float16,0,0.7182559967041016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,96,2,128,1,float16,float16,0,0.38593759536743166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,96,1,128,1,float16,float16,0,0.3656399965286255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,96,4,128,1,float16,float16,0,0.7275968074798584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,96,96,128,1,float16,float16,0,0.3587696075439453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,96,4,128,1,float16,float16,0,0.39058239459991456
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,96,8,128,1,float16,float16,0,0.3928960084915161
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,96,96,128,1,float16,float16,0,0.6695824146270752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,96,8,128,1,float16,float16,0,0.7470560073852539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,96,1,128,1,float16,float16,0,0.20022079944610596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,96,2,128,1,float16,float16,0,0.2002943992614746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,96,1,128,1,float16,float16,0,0.11266720294952393
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,96,2,128,1,float16,float16,0,0.11696799993515014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,96,8,128,1,float16,float16,0,0.21069281101226806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,96,4,128,1,float16,float16,0,0.204201602935791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,96,96,128,1,float16,float16,0,0.19581120014190673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,96,4,128,1,float16,float16,0,0.11923359632492066
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,96,8,128,1,float16,float16,0,0.12120640277862549
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,96,96,128,1,float16,float16,0,0.10988479852676392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,96,1,128,1,float16,float16,0,0.06657119989395141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,96,4,128,1,float16,float16,0,0.0707535982131958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,96,8,128,1,float16,float16,0,0.07862079739570618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,96,2,128,1,float16,float16,0,0.07318879961967469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,96,1,128,1,float16,float16,0,1.8257408142089844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,96,2,128,1,float16,float16,0,1.8264015197753907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,96,4,128,1,float16,float16,0,1.904350471496582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,96,8,128,1,float16,float16,0,1.9542720794677735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,96,1,128,1,float16,float16,0,3.6525745391845703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,96,2,128,1,float16,float16,0,3.7043872833251954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,96,4,128,1,float16,float16,0,3.6429630279541017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,96,1,128,1,float16,float16,0,0.9511360168457031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,96,8,128,1,float16,float16,0,3.9796127319335937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,96,96,128,1,float16,float16,0,4.282643127441406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,96,4,128,1,float16,float16,0,0.9640687942504883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,96,2,128,1,float16,float16,0,0.9516096115112305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,96,1,128,1,float16,float16,0,0.5106031894683838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,96,96,128,1,float16,float16,0,2.1840703964233397
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,96,8,128,1,float16,float16,0,1.0071328163146973
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,96,2,128,1,float16,float16,0,0.4749584197998047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,96,96,128,1,float16,float16,0,1.12424955368042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,96,4,128,1,float16,float16,0,0.4909776210784912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,96,8,128,1,float16,float16,0,0.5065728187561035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,96,2,128,1,float16,float16,0,0.256931209564209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,96,96,128,1,float16,float16,0,0.5828783988952637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,96,1,128,1,float16,float16,0,0.24728479385375976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,96,4,128,1,float16,float16,0,0.25402560234069826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,96,1,128,1,float16,float16,0,0.13742879629135132
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,96,8,128,1,float16,float16,0,0.26103041172027586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,96,96,128,1,float16,float16,0,0.30764000415802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,96,4,128,1,float16,float16,0,0.13915519714355468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,96,1,128,1,float16,float16,0,0.08166239857673645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,96,8,128,1,float16,float16,0,0.14809119701385498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,96,2,128,1,float16,float16,0,0.13929439783096315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,96,2,128,1,float16,float16,0,0.08209279775619507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,96,96,128,1,float16,float16,0,0.16850559711456298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,96,8,128,1,float16,float16,0,0.08515840172767639
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,96,4,128,1,float16,float16,0,0.08218719959259033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,96,96,128,1,float16,float16,0,0.09254080057144165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,96,1,128,1,float16,float16,0,0.050382399559021
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,96,2,128,1,float16,float16,0,0.05063040256500244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,96,8,128,1,float16,float16,0,0.05374079942703247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,96,4,128,1,float16,float16,0,0.051769602298736575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,96,96,128,1,float16,float16,0,0.053260797262191774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,96,1,128,1,float16,float16,0,0.02866080105304718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,96,4,128,1,float16,float16,0,0.030435198545455934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,96,2,128,1,float16,float16,0,0.031497600674629214
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,96,8,128,1,float16,float16,0,0.02879199981689453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,96,1,128,1,float16,float16,0,0.6961872100830078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,96,2,128,1,float16,float16,0,0.7038015842437744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,96,4,128,1,float16,float16,0,0.7364272117614746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,96,1,128,1,float16,float16,0,1.3733823776245118
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,96,2,128,1,float16,float16,0,1.3967120170593261
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,96,4,128,1,float16,float16,0,1.4410287857055664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,96,1,128,1,float16,float16,0,0.35663681030273436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,96,8,128,1,float16,float16,0,0.7776127815246582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,96,8,128,1,float16,float16,0,1.552451229095459
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,96,2,128,1,float16,float16,0,0.3711312055587769
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,96,1,128,1,float16,float16,0,0.18820799589157106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,96,4,128,1,float16,float16,0,0.3843791961669922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,96,96,128,1,float16,float16,0,2.0818304061889648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,96,8,128,1,float16,float16,0,0.40818238258361816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,96,96,128,1,float16,float16,0,1.068400001525879
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,96,2,128,1,float16,float16,0,0.19460959434509278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,96,4,128,1,float16,float16,0,0.20216801166534423
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,96,96,128,1,float16,float16,0,0.5585519790649414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,96,8,128,1,float16,float16,0,0.2112287998199463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,96,96,128,1,float16,float16,0,0.2986687898635864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,96,8,128,1,float16,float16,0,0.11863360404968262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,96,1,128,1,float16,float16,0,0.10448000431060792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,96,2,128,1,float16,float16,0,0.1050271987915039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,96,4,128,1,float16,float16,0,0.10884000062942505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,96,1,128,1,float16,float16,0,0.06412320137023926
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,96,96,128,1,float16,float16,0,0.15555039644241334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,96,2,128,1,float16,float16,0,0.06416640281677247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,96,4,128,1,float16,float16,0,0.06576160192489625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,96,1,128,1,float16,float16,0,0.03531520068645477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,96,8,128,1,float16,float16,0,0.0692575991153717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,96,8,128,1,float16,float16,0,0.040092799067497256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,96,96,128,1,float16,float16,0,0.08353279829025269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,96,4,128,1,float16,float16,0,0.037273600697517395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,96,2,128,1,float16,float16,0,0.03691839873790741
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,96,96,128,1,float16,float16,0,0.04411840140819549
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,96,1,128,1,float16,float16,0,0.023585599660873414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,96,2,128,1,float16,float16,0,0.02330400049686432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,96,4,128,1,float16,float16,0,0.022257600724697114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,96,8,128,1,float16,float16,0,0.023947200179100035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,96,4,128,1,float16,float16,0,0.018670399487018586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,96,1,128,1,float16,float16,0,0.018606400489807128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,96,8,128,1,float16,float16,0,0.019142399728298187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,96,96,128,1,float16,float16,0,0.02433599978685379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,96,2,128,1,float16,float16,0,0.018400000035762788
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,96,1,128,1,float16,float16,0,0.6625455856323242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,96,1,128,1,float16,float16,0,0.3503583908081055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,96,2,128,1,float16,float16,0,0.3544816017150879
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,96,4,128,1,float16,float16,0,0.7553120136260987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,96,2,128,1,float16,float16,0,0.7024879932403565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,96,8,128,1,float16,float16,0,0.8082063674926758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,96,4,128,1,float16,float16,0,0.38893120288848876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,96,1,128,1,float16,float16,0,0.18528319597244264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,96,96,128,1,float16,float16,0,1.0649680137634276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,96,2,128,1,float16,float16,0,0.19092960357666017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,96,4,128,1,float16,float16,0,0.2044111967086792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,96,8,128,1,float16,float16,0,0.42417120933532715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,96,96,128,1,float16,float16,0,0.5461552143096924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,96,8,128,1,float16,float16,0,0.21791520118713378
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,96,2,128,1,float16,float16,0,0.1030351996421814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,96,96,128,1,float16,float16,0,0.2875360012054443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,96,1,128,1,float16,float16,0,0.10087360143661499
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,96,4,128,1,float16,float16,0,0.10675359964370727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,96,8,128,1,float16,float16,0,0.11817439794540405
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,96,1,128,1,float16,float16,0,0.059057599306106566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,96,2,128,1,float16,float16,0,0.06114400029182434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,96,4,128,1,float16,float16,0,0.06232159733772278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,96,96,128,1,float16,float16,0,0.15612000226974487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,96,96,128,1,float16,float16,0,0.08455520272254943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,96,1,128,1,float16,float16,0,0.030745598673820495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,96,8,128,1,float16,float16,0,0.06646720170974732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,96,2,128,1,float16,float16,0,0.03115360140800476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,96,4,128,1,float16,float16,0,0.032419198751449586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,96,8,128,1,float16,float16,0,0.03497920036315918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,96,1,128,1,float16,float16,0,0.02050720006227493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,96,2,128,1,float16,float16,0,0.02044160068035126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,96,96,128,1,float16,float16,0,0.04708159863948822
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,96,8,128,1,float16,float16,0,0.020095999538898467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,96,4,128,1,float16,float16,0,0.019828799366950988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,96,96,128,1,float16,float16,0,0.02311680018901825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,96,1,128,1,float16,float16,0,0.01693280041217804
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,96,4,128,1,float16,float16,0,0.016574400663375854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,96,2,128,1,float16,float16,0,0.016971200704574585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,96,8,128,1,float16,float16,0,0.016436800360679626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,96,96,128,1,float16,float16,0,0.018436799943447112
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,96,2,128,1,float16,float16,0,0.015321600437164306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,96,4,128,1,float16,float16,0,0.015385599434375763
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,96,1,128,1,float16,float16,0,0.01541920006275177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,96,8,128,1,float16,float16,0,0.015382400155067444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,96,1,128,1,float16,float16,0,0.32837440967559817
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,96,2,128,1,float16,float16,0,0.18007199764251708
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,96,1,128,1,float16,float16,0,0.1749392032623291
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,96,4,128,1,float16,float16,0,0.36236159801483153
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,96,2,128,1,float16,float16,0,0.3409728050231934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,96,8,128,1,float16,float16,0,0.3865855932235718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,96,4,128,1,float16,float16,0,0.18986239433288574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,96,1,128,1,float16,float16,0,0.10043519735336304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,96,8,128,1,float16,float16,0,0.20428800582885742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,96,96,128,1,float16,float16,0,0.6993711948394775
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,96,2,128,1,float16,float16,0,0.10189919471740723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,96,4,128,1,float16,float16,0,0.10618239641189575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,96,96,128,1,float16,float16,0,0.35677599906921387
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,96,8,128,1,float16,float16,0,0.11198559999465943
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,96,1,128,1,float16,float16,0,0.0598143994808197
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,96,2,128,1,float16,float16,0,0.06071360111236572
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,96,4,128,1,float16,float16,0,0.06208800077438355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,96,8,128,1,float16,float16,0,0.06511840224266052
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,96,96,128,1,float16,float16,0,0.18493759632110596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,96,1,128,1,float16,float16,0,0.03099679946899414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,96,2,128,1,float16,float16,0,0.0312032014131546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,96,96,128,1,float16,float16,0,0.10083680152893067
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,96,8,128,1,float16,float16,0,0.03522560000419617
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,96,1,128,1,float16,float16,0,0.019998399913311003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,96,4,128,1,float16,float16,0,0.032308799028396604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,96,96,128,1,float16,float16,0,0.05899999737739563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,96,2,128,1,float16,float16,0,0.01974560022354126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,96,4,128,1,float16,float16,0,0.01958560049533844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,96,2,128,1,float16,float16,0,0.016441600024700166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,96,1,128,1,float16,float16,0,0.016631999611854555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,96,8,128,1,float16,float16,0,0.01979839950799942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,96,96,128,1,float16,float16,0,0.029151999950408937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,96,96,128,1,float16,float16,0,0.019944000244140624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,96,4,128,1,float16,float16,0,0.016305600106716157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,96,8,128,1,float16,float16,0,0.01653439998626709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,96,1,128,1,float16,float16,0,0.015161600708961488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,96,2,128,1,float16,float16,0,0.01510079950094223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,96,1,128,1,float16,float16,0,0.01496800035238266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,96,96,128,1,float16,float16,0,0.01777919977903366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,96,8,128,1,float16,float16,0,0.015358400344848634
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,96,4,128,1,float16,float16,0,0.015118399262428283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,96,2,128,1,float16,float16,0,0.014876799285411834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,96,4,128,1,float16,float16,0,0.014998400211334228
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,96,8,128,1,float16,float16,0,0.015137599408626556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,96,2,128,1,float16,float16,0,0.17428159713745117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,96,8,128,1,float16,float16,0,0.22224318981170654
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,96,1,128,1,float16,float16,0,0.17267680168151855
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,96,1,128,1,float16,float16,0,0.09914720058441162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,96,4,128,1,float16,float16,0,0.18400800228118896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,96,2,128,1,float16,float16,0,0.10145599842071533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,96,4,128,1,float16,float16,0,0.10352480411529541
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,96,8,128,1,float16,float16,0,0.12308640480041504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,96,1,128,1,float16,float16,0,0.06154559850692749
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,96,2,128,1,float16,float16,0,0.06133120059967041
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,96,4,128,1,float16,float16,0,0.06367200016975402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,96,8,128,1,float16,float16,0,0.07119519710540771
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,96,96,128,1,float16,float16,0,0.5673696041107178
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,96,1,128,1,float16,float16,0,0.03229599893093109
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,96,96,128,1,float16,float16,0,0.2885119915008545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,96,4,128,1,float16,float16,0,0.03239679932594299
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,96,2,128,1,float16,float16,0,0.032734400033950804
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,96,96,128,1,float16,float16,0,0.1511855959892273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,96,96,128,1,float16,float16,0,0.08485599756240844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,96,2,128,1,float16,float16,0,0.019617600739002226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,96,1,128,1,float16,float16,0,0.020139199495315552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,96,4,128,1,float16,float16,0,0.019441600143909454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,96,8,128,1,float16,float16,0,0.02475679963827133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,96,8,128,1,float16,float16,0,0.04349919855594635
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,96,1,128,1,float16,float16,0,0.016359999775886536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,96,2,128,1,float16,float16,0,0.016318400204181672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,96,96,128,1,float16,float16,0,0.04376319944858551
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,96,8,128,1,float16,float16,0,0.018105599284172057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,96,4,128,1,float16,float16,0,0.016276800632476808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,96,1,128,1,float16,float16,0,0.015182399749755859
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,96,96,128,1,float16,float16,0,0.0272816002368927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,96,2,128,1,float16,float16,0,0.015294399857521058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,96,4,128,1,float16,float16,0,0.015249599516391755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,96,8,128,1,float16,float16,0,0.015214399993419647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,96,96,128,1,float16,float16,0,0.01945119947195053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,96,4,128,1,float16,float16,0,0.01520799994468689
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,96,1,128,1,float16,float16,0,0.01496639996767044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,96,2,128,1,float16,float16,0,0.014963200688362122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,96,96,128,1,float16,float16,0,0.017526400089263917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,96,1,128,1,float16,float16,0,0.014628799259662628
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,96,8,128,1,float16,float16,0,0.015041600167751312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,96,2,128,1,float16,float16,0,0.014726400375366211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,96,4,128,1,float16,float16,0,0.014763200283050537
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,96,1,128,1,float16,float16,0,0.028755199909210206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,96,8,128,1,float16,float16,0,0.015068799257278442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,96,4,128,1,float16,float16,0,0.05257599949836731
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,96,2,128,1,float16,float16,0,0.03506399989128113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,96,8,128,1,float16,float16,0,0.08960000276565552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,96,1,128,1,float16,float16,0,0.019497600197792054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,96,2,128,1,float16,float16,0,0.02521120011806488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,96,8,128,1,float16,float16,0,0.049979200959205626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,96,4,128,1,float16,float16,0,0.03251680135726929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,96,1,128,1,float16,float16,0,0.016124799847602844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,96,4,128,1,float16,float16,0,0.023647999763488768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,96,96,128,1,float16,float16,0,0.22335519790649414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,96,8,128,1,float16,float16,0,0.03116639852523804
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,96,2,128,1,float16,float16,0,0.017892800271511078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,96,96,128,1,float16,float16,0,0.4336239814758301
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,96,96,128,1,float16,float16,0,0.11896480321884155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,96,1,128,1,float16,float16,0,0.0152879998087883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,96,2,128,1,float16,float16,0,0.015254400670528412
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,96,4,128,1,float16,float16,0,0.017127999663352968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,96,96,128,1,float16,float16,0,0.06729919910430908
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,96,1,128,1,float16,float16,0,0.014555199444293976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,96,2,128,1,float16,float16,0,0.014323200285434722
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,96,8,128,1,float16,float16,0,0.022779199481010436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,96,4,128,1,float16,float16,0,0.014670400321483612
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,96,8,128,1,float16,float16,0,0.01640959978103638
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,96,96,128,1,float16,float16,0,0.04079839885234833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,96,2,128,1,float16,float16,0,0.014388799667358398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,96,1,128,1,float16,float16,0,0.014238399267196656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,96,8,128,1,float16,float16,0,0.014478400349617004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,96,96,128,1,float16,float16,0,0.02643679976463318
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,96,4,128,1,float16,float16,0,0.014472000300884247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,96,1,128,1,float16,float16,0,0.014412799477577209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,96,2,128,1,float16,float16,0,0.014343999326229095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,96,4,128,1,float16,float16,0,0.01401119977235794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,96,8,128,1,float16,float16,0,0.014251199364662171
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,96,96,128,1,float16,float16,0,0.01902880072593689
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,96,2,128,1,float16,float16,0,0.01417279988527298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,96,1,128,1,float16,float16,0,0.014239999651908874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,96,8,128,1,float16,float16,0,0.014132800698280334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,96,4,128,1,float16,float16,0,0.01414559930562973
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,96,96,128,1,float16,float16,0,0.017041599750518797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,96,1,128,1,float16,float16,0,0.014155200123786927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,96,2,128,1,float16,float16,0,0.01401599943637848
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,96,8,128,1,float16,float16,0,0.01395840048789978
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,96,4,128,1,float16,float16,0,0.014239999651908874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,64,1,128,1,float16,float16,0,84.02161865234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,64,2,128,1,float16,float16,0,84.17822265625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,64,4,128,1,float16,float16,0,86.4852783203125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,64,1,128,1,float16,float16,0,43.30965881347656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,64,1,128,1,float16,float16,0,167.79254150390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,64,8,128,1,float16,float16,0,84.33895263671874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,64,4,128,1,float16,float16,0,170.4067626953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,64,2,128,1,float16,float16,0,170.58272705078124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,64,8,128,1,float16,float16,0,170.899072265625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,64,2,128,1,float16,float16,0,43.36163330078125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,64,1,128,1,float16,float16,0,21.479777526855468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,64,2,128,1,float16,float16,0,21.507762145996093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,64,4,128,1,float16,float16,0,21.847601318359374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,64,4,128,1,float16,float16,0,42.25693054199219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,64,8,128,1,float16,float16,0,42.328802490234374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,64,8,128,1,float16,float16,0,21.72528839111328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,64,64,128,1,float16,float16,0,22.49364013671875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,64,1,128,1,float16,float16,0,96.366796875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,64,2,128,1,float16,float16,0,97.665185546875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,64,64,128,1,float16,float16,0,44.160885620117185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,64,4,128,1,float16,float16,0,95.3225341796875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,64,8,128,1,float16,float16,0,95.4480224609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,64,64,128,1,float16,float16,0,50.9978759765625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,64,1,128,1,float16,float16,0,48.055731201171874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,64,64,128,1,float16,float16,0,25.298239135742186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,64,2,128,1,float16,float16,0,48.338201904296874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,64,64,128,1,float16,float16,0,88.53031005859376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,64,1,128,1,float16,float16,0,23.755078125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,64,2,128,1,float16,float16,0,24.12184295654297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,64,4,128,1,float16,float16,0,49.07959899902344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,64,8,128,1,float16,float16,0,47.804769897460936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,64,2,128,1,float16,float16,0,12.033995056152344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,64,1,128,1,float16,float16,0,11.994010925292969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,64,4,128,1,float16,float16,0,12.220225524902343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,64,64,128,1,float16,float16,0,12.996783447265624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,64,4,128,1,float16,float16,0,24.125885009765625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,64,8,128,1,float16,float16,0,23.981256103515626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,64,8,128,1,float16,float16,0,12.400278472900391
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,64,1,128,1,float16,float16,0,33.40430603027344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,64,2,128,1,float16,float16,0,33.227047729492185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,64,4,128,1,float16,float16,0,34.35478210449219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,64,64,128,1,float16,float16,0,35.97297973632813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,64,64,128,1,float16,float16,0,17.841400146484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,64,1,128,1,float16,float16,0,17.032974243164062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,64,2,128,1,float16,float16,0,16.853306579589844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,64,1,128,1,float16,float16,0,66.2144287109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,64,64,128,1,float16,float16,0,9.130941009521484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,64,2,128,1,float16,float16,0,66.26436767578124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,64,4,128,1,float16,float16,0,66.9828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,64,1,128,1,float16,float16,0,8.32855682373047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,64,8,128,1,float16,float16,0,67.6323486328125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,64,8,128,1,float16,float16,0,33.68394470214844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,64,4,128,1,float16,float16,0,16.677198791503905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,64,8,128,1,float16,float16,0,17.355863952636717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,64,2,128,1,float16,float16,0,8.497537231445312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,64,4,128,1,float16,float16,0,8.408860778808593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,64,8,128,1,float16,float16,0,8.508972930908204
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,64,1,128,1,float16,float16,0,43.28053283691406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,64,2,128,1,float16,float16,0,42.692266845703124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,64,4,128,1,float16,float16,0,43.19142456054688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,64,64,128,1,float16,float16,0,47.204287719726565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,64,1,128,1,float16,float16,0,21.483282470703124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,64,64,128,1,float16,float16,0,22.98808898925781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,64,2,128,1,float16,float16,0,21.823701477050783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,64,1,128,1,float16,float16,0,85.22396240234374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,64,2,128,1,float16,float16,0,86.45193481445312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,64,4,128,1,float16,float16,0,86.25145874023437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,64,8,128,1,float16,float16,0,85.7530517578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,64,64,128,1,float16,float16,0,12.077942657470704
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,64,8,128,1,float16,float16,0,43.65528869628906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,64,4,128,1,float16,float16,0,21.76733856201172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,64,1,128,1,float16,float16,0,10.697150421142577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,64,64,128,1,float16,float16,0,6.037027359008789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,64,2,128,1,float16,float16,0,10.860377502441406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,64,8,128,1,float16,float16,0,21.56084747314453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,64,1,128,1,float16,float16,0,5.475846481323242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,64,4,128,1,float16,float16,0,10.932917022705078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,64,8,128,1,float16,float16,0,10.85556640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,64,2,128,1,float16,float16,0,5.437606430053711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,64,4,128,1,float16,float16,0,5.435990524291992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,64,8,128,1,float16,float16,0,5.605632019042969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,64,1,128,1,float16,float16,0,24.372601318359376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,64,2,128,1,float16,float16,0,24.33839111328125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,64,64,128,1,float16,float16,0,26.68538818359375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,64,4,128,1,float16,float16,0,25.089923095703124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,64,1,128,1,float16,float16,0,12.117708587646485
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,64,64,128,1,float16,float16,0,13.422984313964843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,64,2,128,1,float16,float16,0,12.479431915283204
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,64,1,128,1,float16,float16,0,49.1644775390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,64,2,128,1,float16,float16,0,48.468817138671874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,64,64,128,1,float16,float16,0,6.892639923095703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,64,4,128,1,float16,float16,0,49.365020751953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,64,8,128,1,float16,float16,0,49.3713134765625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,64,8,128,1,float16,float16,0,24.69452362060547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,64,4,128,1,float16,float16,0,12.176197052001953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,64,64,128,1,float16,float16,0,3.4361873626708985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,64,8,128,1,float16,float16,0,12.259081268310547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,64,1,128,1,float16,float16,0,3.0647951126098634
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,64,1,128,1,float16,float16,0,6.166523361206055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,64,2,128,1,float16,float16,0,6.121441650390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,64,2,128,1,float16,float16,0,3.0742063522338867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,64,4,128,1,float16,float16,0,6.276176071166992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,64,8,128,1,float16,float16,0,6.298791885375977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,64,4,128,1,float16,float16,0,3.1279855728149415
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,64,8,128,1,float16,float16,0,3.1584144592285157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,64,1,128,1,float16,float16,0,22.28996124267578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,64,2,128,1,float16,float16,0,22.112229919433595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,64,4,128,1,float16,float16,0,22.085736083984376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,64,64,128,1,float16,float16,0,25.69371337890625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,64,1,128,1,float16,float16,0,11.016220855712891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,64,64,128,1,float16,float16,0,12.746617889404297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,64,2,128,1,float16,float16,0,11.29349136352539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,64,1,128,1,float16,float16,0,44.254129028320314
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,64,4,128,1,float16,float16,0,44.261514282226564
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,64,2,128,1,float16,float16,0,44.59769897460937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,64,8,128,1,float16,float16,0,44.327203369140626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,64,64,128,1,float16,float16,0,6.415996551513672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,64,8,128,1,float16,float16,0,22.644972229003905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,64,4,128,1,float16,float16,0,11.065988922119141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,64,8,128,1,float16,float16,0,11.211036682128906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,64,64,128,1,float16,float16,0,3.2351215362548826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,64,1,128,1,float16,float16,0,2.8172815322875975
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,64,1,128,1,float16,float16,0,5.4776863098144535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,64,2,128,1,float16,float16,0,2.7513967514038087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,64,2,128,1,float16,float16,0,5.494214248657227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,64,4,128,1,float16,float16,0,5.651692962646484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,64,8,128,1,float16,float16,0,5.602993774414062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,64,1,128,1,float16,float16,0,1.4291199684143066
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,64,64,128,1,float16,float16,0,1.6685487747192382
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,64,4,128,1,float16,float16,0,2.8143808364868166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,64,2,128,1,float16,float16,0,1.4288911819458008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,64,8,128,1,float16,float16,0,2.8517200469970705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,64,4,128,1,float16,float16,0,1.4471280097961425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,64,8,128,1,float16,float16,0,1.4820032119750977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,64,1,128,1,float16,float16,0,12.732902526855469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,64,2,128,1,float16,float16,0,12.547618865966797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,64,64,128,1,float16,float16,0,15.22928466796875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,64,4,128,1,float16,float16,0,12.865446472167969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,64,64,128,1,float16,float16,0,7.378614044189453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,64,1,128,1,float16,float16,0,6.434480285644531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,64,1,128,1,float16,float16,0,25.0973388671875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,64,2,128,1,float16,float16,0,25.41291198730469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,64,4,128,1,float16,float16,0,25.650210571289062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,64,8,128,1,float16,float16,0,25.637759399414062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,64,8,128,1,float16,float16,0,12.920579528808593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,64,2,128,1,float16,float16,0,6.424459075927734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,64,4,128,1,float16,float16,0,6.349164962768555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,64,1,128,1,float16,float16,0,3.183734321594238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,64,64,128,1,float16,float16,0,3.7399486541748046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,64,2,128,1,float16,float16,0,3.178676795959473
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,64,4,128,1,float16,float16,0,3.1839391708374025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,64,8,128,1,float16,float16,0,6.571451568603516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,64,64,128,1,float16,float16,0,1.9230432510375977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,64,8,128,1,float16,float16,0,3.2855712890625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,64,1,128,1,float16,float16,0,0.8672112464904785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,64,1,128,1,float16,float16,0,1.6019472122192382
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,64,64,128,1,float16,float16,0,1.010209560394287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,64,2,128,1,float16,float16,0,1.6050735473632813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,64,2,128,1,float16,float16,0,0.859284782409668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,64,8,128,1,float16,float16,0,1.6778160095214845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,64,4,128,1,float16,float16,0,1.6297887802124023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,64,4,128,1,float16,float16,0,0.8829248428344727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,64,8,128,1,float16,float16,0,0.875487995147705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,64,1,128,1,float16,float16,0,11.571723175048827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,64,2,128,1,float16,float16,0,11.738130950927735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,64,4,128,1,float16,float16,0,11.998175811767577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,64,64,128,1,float16,float16,0,14.634071350097656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,64,1,128,1,float16,float16,0,5.907100677490234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,64,64,128,1,float16,float16,0,7.4309326171875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,64,2,128,1,float16,float16,0,5.842022323608399
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,64,1,128,1,float16,float16,0,23.07722625732422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,64,2,128,1,float16,float16,0,23.20526580810547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,64,4,128,1,float16,float16,0,23.498956298828126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,64,8,128,1,float16,float16,0,11.847147369384766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,64,8,128,1,float16,float16,0,24.118183898925782
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,64,64,128,1,float16,float16,0,3.645161437988281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,64,4,128,1,float16,float16,0,6.0019489288330075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,64,1,128,1,float16,float16,0,2.8559167861938475
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,64,2,128,1,float16,float16,0,2.881892776489258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,64,64,128,1,float16,float16,0,1.8621824264526368
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,64,8,128,1,float16,float16,0,6.006156921386719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,64,4,128,1,float16,float16,0,2.917156791687012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,64,1,128,1,float16,float16,0,1.4501296043395997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,64,2,128,1,float16,float16,0,1.491220760345459
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,64,8,128,1,float16,float16,0,2.9792448043823243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,64,1,128,1,float16,float16,0,0.7663040161132812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,64,64,128,1,float16,float16,0,0.9647295951843262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,64,2,128,1,float16,float16,0,0.7974224090576172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,64,4,128,1,float16,float16,0,1.4996447563171387
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,64,64,128,1,float16,float16,0,0.5218128204345703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,64,8,128,1,float16,float16,0,1.509718418121338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,64,4,128,1,float16,float16,0,0.7764800071716309
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,64,1,128,1,float16,float16,0,0.4037312030792236
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,64,8,128,1,float16,float16,0,0.8041184425354004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,64,2,128,1,float16,float16,0,0.4128736019134521
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,64,4,128,1,float16,float16,0,0.4367216110229492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,64,8,128,1,float16,float16,0,0.4429135799407959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,64,2,128,1,float16,float16,0,6.896144104003906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,64,1,128,1,float16,float16,0,6.859130859375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,64,4,128,1,float16,float16,0,6.798381042480469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,64,64,128,1,float16,float16,0,9.125096130371094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,64,1,128,1,float16,float16,0,3.3657886505126955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,64,64,128,1,float16,float16,0,4.519575881958008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,64,2,128,1,float16,float16,0,3.4588817596435546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,64,1,128,1,float16,float16,0,13.441976928710938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,64,2,128,1,float16,float16,0,13.582574462890625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,64,4,128,1,float16,float16,0,13.736053466796875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,64,8,128,1,float16,float16,0,14.076658630371094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,64,8,128,1,float16,float16,0,6.949729919433594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,64,4,128,1,float16,float16,0,3.386433410644531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,64,1,128,1,float16,float16,0,1.678513526916504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,64,2,128,1,float16,float16,0,1.6838272094726563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,64,1,128,1,float16,float16,0,0.8666447639465332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,64,64,128,1,float16,float16,0,2.2712879180908203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,64,8,128,1,float16,float16,0,3.5282047271728514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,64,64,128,1,float16,float16,0,1.1540767669677734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,64,4,128,1,float16,float16,0,1.7154560089111328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,64,8,128,1,float16,float16,0,1.7249216079711913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,64,2,128,1,float16,float16,0,0.8835904121398925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,64,2,128,1,float16,float16,0,0.46830239295959475
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,64,1,128,1,float16,float16,0,0.4662655830383301
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,64,4,128,1,float16,float16,0,0.48631677627563474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,64,4,128,1,float16,float16,0,0.879361629486084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,64,64,128,1,float16,float16,0,0.6169328212738037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,64,64,128,1,float16,float16,0,0.3388688087463379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,64,8,128,1,float16,float16,0,0.4950736045837402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,64,8,128,1,float16,float16,0,0.8777279853820801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,64,1,128,1,float16,float16,0,0.2524192094802856
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,64,2,128,1,float16,float16,0,0.24923999309539796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,64,4,128,1,float16,float16,0,0.2669392108917236
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,64,8,128,1,float16,float16,0,0.27403199672698975
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,64,1,128,1,float16,float16,0,6.355534362792969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,64,2,128,1,float16,float16,0,6.4820098876953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,64,64,128,1,float16,float16,0,9.392131042480468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,64,1,128,1,float16,float16,0,12.83387451171875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,64,4,128,1,float16,float16,0,12.956379699707032
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,64,2,128,1,float16,float16,0,13.001658630371093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,64,1,128,1,float16,float16,0,3.2163551330566404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,64,8,128,1,float16,float16,0,13.399240112304687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,64,4,128,1,float16,float16,0,6.506449890136719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,64,64,128,1,float16,float16,0,4.631718444824219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,64,8,128,1,float16,float16,0,6.817665863037109
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,64,1,128,1,float16,float16,0,1.5802687644958495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,64,2,128,1,float16,float16,0,3.17633113861084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,64,2,128,1,float16,float16,0,1.5688015937805175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,64,4,128,1,float16,float16,0,1.600774383544922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,64,8,128,1,float16,float16,0,1.652134323120117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,64,4,128,1,float16,float16,0,3.2353328704833983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,64,64,128,1,float16,float16,0,2.3209184646606444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,64,64,128,1,float16,float16,0,1.1926863670349122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,64,8,128,1,float16,float16,0,3.310019302368164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,64,1,128,1,float16,float16,0,0.8143775939941407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,64,2,128,1,float16,float16,0,0.8163999557495117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,64,4,128,1,float16,float16,0,0.8270912170410156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,64,64,128,1,float16,float16,0,0.6289680004119873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,64,8,128,1,float16,float16,0,0.852137565612793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,64,1,128,1,float16,float16,0,0.4332320213317871
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,64,2,128,1,float16,float16,0,0.43381600379943847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,64,64,128,1,float16,float16,0,0.3289119958877563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,64,4,128,1,float16,float16,0,0.44727840423583987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,64,1,128,1,float16,float16,0,0.23659040927886962
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,64,8,128,1,float16,float16,0,0.45116801261901857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,64,4,128,1,float16,float16,0,0.25152480602264404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,64,2,128,1,float16,float16,0,0.24334239959716797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,64,8,128,1,float16,float16,0,0.24954240322113036
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,64,64,128,1,float16,float16,0,0.19486080408096312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,64,1,128,1,float16,float16,0,0.13914400339126587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,64,4,128,1,float16,float16,0,0.13759679794311525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,64,2,128,1,float16,float16,0,0.14098559617996215
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,64,8,128,1,float16,float16,0,0.14928959608078002
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,64,1,128,1,float16,float16,0,3.771015930175781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,64,2,128,1,float16,float16,0,3.8059951782226564
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,64,4,128,1,float16,float16,0,3.8552177429199217
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,64,1,128,1,float16,float16,0,1.8741567611694336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,64,64,128,1,float16,float16,0,6.610142517089844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,64,1,128,1,float16,float16,0,7.789347076416016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,64,2,128,1,float16,float16,0,7.661402893066406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,64,4,128,1,float16,float16,0,7.9468544006347654
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,64,8,128,1,float16,float16,0,8.176822662353516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,64,8,128,1,float16,float16,0,4.098011016845703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,64,64,128,1,float16,float16,0,3.3894222259521483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,64,2,128,1,float16,float16,0,1.8910127639770509
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,64,4,128,1,float16,float16,0,1.9033504486083985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,64,64,128,1,float16,float16,0,1.708892822265625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,64,2,128,1,float16,float16,0,0.961956787109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,64,4,128,1,float16,float16,0,0.9829487800598145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,64,1,128,1,float16,float16,0,0.9502736091613769
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,64,8,128,1,float16,float16,0,2.0647232055664064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,64,8,128,1,float16,float16,0,1.0090767860412597
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,64,64,128,1,float16,float16,0,0.8844367980957031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,64,1,128,1,float16,float16,0,0.49459199905395507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,64,2,128,1,float16,float16,0,0.49401440620422366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,64,2,128,1,float16,float16,0,0.2681488037109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,64,4,128,1,float16,float16,0,0.5032127857208252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,64,4,128,1,float16,float16,0,0.27031359672546384
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,64,1,128,1,float16,float16,0,0.275982403755188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,64,8,128,1,float16,float16,0,0.2766767978668213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,64,64,128,1,float16,float16,0,0.4589087963104248
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,64,64,128,1,float16,float16,0,0.2478543996810913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,64,8,128,1,float16,float16,0,0.51387038230896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,64,2,128,1,float16,float16,0,0.14835200309753419
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,64,1,128,1,float16,float16,0,0.14444320201873778
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,64,4,128,1,float16,float16,0,0.14997919797897338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,64,2,128,1,float16,float16,0,0.08669760227203369
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,64,1,128,1,float16,float16,0,0.08636159896850586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,64,8,128,1,float16,float16,0,0.15841920375823976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,64,64,128,1,float16,float16,0,0.13772000074386598
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,64,8,128,1,float16,float16,0,0.09268959760665893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,64,4,128,1,float16,float16,0,0.0930400013923645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,64,1,128,1,float16,float16,0,0.05819360017776489
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,64,64,128,1,float16,float16,0,0.08227999806404114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,64,2,128,1,float16,float16,0,0.05399360060691834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,64,4,128,1,float16,float16,0,0.052692800760269165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,64,8,128,1,float16,float16,0,0.058430397510528566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,64,1,128,1,float16,float16,0,1.2291296005249024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,64,2,128,1,float16,float16,0,1.240931224822998
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,64,4,128,1,float16,float16,0,1.2744848251342773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,64,2,128,1,float16,float16,0,2.495083236694336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,64,1,128,1,float16,float16,0,2.4503200531005858
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,64,8,128,1,float16,float16,0,1.360915184020996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,64,1,128,1,float16,float16,0,0.660868787765503
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,64,4,128,1,float16,float16,0,2.576358413696289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,64,8,128,1,float16,float16,0,2.6686559677124024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,64,64,128,1,float16,float16,0,2.8329200744628906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,64,2,128,1,float16,float16,0,0.6641679763793945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,64,1,128,1,float16,float16,0,0.3248303890228271
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,64,64,128,1,float16,float16,0,1.458790397644043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,64,4,128,1,float16,float16,0,0.684876823425293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,64,8,128,1,float16,float16,0,0.6935743808746337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,64,64,128,1,float16,float16,0,0.7425392150878907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,64,2,128,1,float16,float16,0,0.3318048000335693
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,64,4,128,1,float16,float16,0,0.3448031902313232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,64,2,128,1,float16,float16,0,0.176528000831604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,64,1,128,1,float16,float16,0,0.17977919578552246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,64,4,128,1,float16,float16,0,0.1822111964225769
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,64,8,128,1,float16,float16,0,0.3643487930297852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,64,8,128,1,float16,float16,0,0.1902240037918091
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,64,64,128,1,float16,float16,0,0.2078927993774414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,64,64,128,1,float16,float16,0,0.39046239852905273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,64,1,128,1,float16,float16,0,0.10021599531173705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,64,2,128,1,float16,float16,0,0.10217119455337524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,64,4,128,1,float16,float16,0,0.10331200361251831
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,64,1,128,1,float16,float16,0,0.06133120059967041
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,64,8,128,1,float16,float16,0,0.11016319990158081
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,64,4,128,1,float16,float16,0,0.06329119801521302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,64,2,128,1,float16,float16,0,0.06369439959526062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,64,8,128,1,float16,float16,0,0.06631839871406556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,64,64,128,1,float16,float16,0,0.11705440282821655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,64,1,128,1,float16,float16,0,0.04095839858055115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,64,64,128,1,float16,float16,0,0.06607840061187745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,64,4,128,1,float16,float16,0,0.03919520080089569
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,64,2,128,1,float16,float16,0,0.041099199652671815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,64,8,128,1,float16,float16,0,0.03924799859523773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,64,2,128,1,float16,float16,0,0.027801600098609925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,64,64,128,1,float16,float16,0,0.039587199687957764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,64,4,128,1,float16,float16,0,0.02977760136127472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,64,8,128,1,float16,float16,0,0.029555198550224305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,64,1,128,1,float16,float16,0,0.026447999477386474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,64,1,128,1,float16,float16,0,0.47267680168151854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,64,2,128,1,float16,float16,0,0.4906000137329102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,64,1,128,1,float16,float16,0,0.9285615921020508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,64,4,128,1,float16,float16,0,0.9989904403686524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,64,2,128,1,float16,float16,0,0.9500831604003906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,64,4,128,1,float16,float16,0,0.5150303840637207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,64,8,128,1,float16,float16,0,1.1068736076354981
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,64,1,128,1,float16,float16,0,0.24956800937652587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,64,64,128,1,float16,float16,0,1.3747664451599122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,64,2,128,1,float16,float16,0,0.2567471981048584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,64,8,128,1,float16,float16,0,0.5744768142700195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,64,4,128,1,float16,float16,0,0.26900320053100585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,64,64,128,1,float16,float16,0,0.7113264083862305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,64,8,128,1,float16,float16,0,0.30500640869140627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,64,1,128,1,float16,float16,0,0.13541439771652222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,64,2,128,1,float16,float16,0,0.13893920183181763
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,64,64,128,1,float16,float16,0,0.3664207935333252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,64,4,128,1,float16,float16,0,0.14720799922943115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,64,8,128,1,float16,float16,0,0.15992319583892822
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,64,4,128,1,float16,float16,0,0.08366079926490784
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,64,1,128,1,float16,float16,0,0.0790831983089447
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,64,64,128,1,float16,float16,0,0.19708160161972046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,64,8,128,1,float16,float16,0,0.09047039747238159
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,64,2,128,1,float16,float16,0,0.08004800081253052
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,64,64,128,1,float16,float16,0,0.10811680555343628
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,64,1,128,1,float16,float16,0,0.050547200441360476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,64,4,128,1,float16,float16,0,0.052820801734924316
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,64,2,128,1,float16,float16,0,0.051560002565383914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,64,64,128,1,float16,float16,0,0.05956320166587829
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,64,8,128,1,float16,float16,0,0.055550402402877806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,64,2,128,1,float16,float16,0,0.030294400453567506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,64,1,128,1,float16,float16,0,0.02990719974040985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,64,4,128,1,float16,float16,0,0.029758399724960326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,64,8,128,1,float16,float16,0,0.030737599730491637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,64,1,128,1,float16,float16,0,0.021454399824142455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,64,64,128,1,float16,float16,0,0.030793601274490358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,64,2,128,1,float16,float16,0,0.022737599909305573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,64,4,128,1,float16,float16,0,0.02295359969139099
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,64,8,128,1,float16,float16,0,0.021844799816608428
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,64,64,128,1,float16,float16,0,0.021241599321365358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,64,2,128,1,float16,float16,0,0.018033599853515624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,64,4,128,1,float16,float16,0,0.018240000307559966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,64,8,128,1,float16,float16,0,0.01894879937171936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,64,1,128,1,float16,float16,0,0.017846399545669557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,64,1,128,1,float16,float16,0,0.24556798934936525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,64,2,128,1,float16,float16,0,0.47638401985168455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,64,2,128,1,float16,float16,0,0.24766719341278076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,64,1,128,1,float16,float16,0,0.4728831768035889
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,64,4,128,1,float16,float16,0,0.5098800182342529
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,64,8,128,1,float16,float16,0,0.5854000091552735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,64,4,128,1,float16,float16,0,0.2681504011154175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,64,1,128,1,float16,float16,0,0.13593759536743164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,64,64,128,1,float16,float16,0,0.7108176231384278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,64,2,128,1,float16,float16,0,0.13486080169677733
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,64,8,128,1,float16,float16,0,0.3058495998382568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,64,4,128,1,float16,float16,0,0.1441007971763611
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,64,64,128,1,float16,float16,0,0.36606559753417967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,64,8,128,1,float16,float16,0,0.16195839643478394
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,64,8,128,1,float16,float16,0,0.08871039748191833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,64,2,128,1,float16,float16,0,0.07591840028762817
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,64,1,128,1,float16,float16,0,0.07711039781570435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,64,4,128,1,float16,float16,0,0.08089600205421447
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,64,64,128,1,float16,float16,0,0.1961567997932434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,64,64,128,1,float16,float16,0,0.10890079736709594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,64,1,128,1,float16,float16,0,0.04681279957294464
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,64,2,128,1,float16,float16,0,0.04779199957847595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,64,4,128,1,float16,float16,0,0.04862079918384552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,64,1,128,1,float16,float16,0,0.026918399333953857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,64,64,128,1,float16,float16,0,0.05924479961395264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,64,2,128,1,float16,float16,0,0.02751680016517639
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,64,8,128,1,float16,float16,0,0.05203679800033569
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,64,4,128,1,float16,float16,0,0.027051201462745665
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,64,8,128,1,float16,float16,0,0.028299200534820556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,64,64,128,1,float16,float16,0,0.030399999022483824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,64,2,128,1,float16,float16,0,0.018475200235843658
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,64,4,128,1,float16,float16,0,0.01876640021800995
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,64,8,128,1,float16,float16,0,0.0191103994846344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,64,1,128,1,float16,float16,0,0.01852640062570572
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,64,64,128,1,float16,float16,0,0.019760000705718993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,64,1,128,1,float16,float16,0,0.01607999950647354
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,64,2,128,1,float16,float16,0,0.01555359959602356
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,64,4,128,1,float16,float16,0,0.01568640023469925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,64,8,128,1,float16,float16,0,0.016288000345230102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,64,64,128,1,float16,float16,0,0.016993600130081176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,64,8,128,1,float16,float16,0,0.015252800285816192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,64,2,128,1,float16,float16,0,0.015196800231933594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,64,1,128,1,float16,float16,0,0.015255999565124512
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,64,4,128,1,float16,float16,0,0.015345600247383118
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,64,1,128,1,float16,float16,0,0.23105440139770508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,64,2,128,1,float16,float16,0,0.2376688003540039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,64,1,128,1,float16,float16,0,0.12886719703674315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,64,4,128,1,float16,float16,0,0.25294721126556396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,64,2,128,1,float16,float16,0,0.1328544020652771
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,64,4,128,1,float16,float16,0,0.13747040033340455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,64,8,128,1,float16,float16,0,0.15148639678955078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,64,8,128,1,float16,float16,0,0.2839951992034912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,64,64,128,1,float16,float16,0,0.4713888168334961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,64,1,128,1,float16,float16,0,0.07492480278015137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,64,4,128,1,float16,float16,0,0.08005599975585938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,64,2,128,1,float16,float16,0,0.07772480249404908
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,64,8,128,1,float16,float16,0,0.08732159733772278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,64,64,128,1,float16,float16,0,0.2461024045944214
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,64,4,128,1,float16,float16,0,0.04917919933795929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,64,2,128,1,float16,float16,0,0.04779039919376373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,64,8,128,1,float16,float16,0,0.05191519856452942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,64,1,128,1,float16,float16,0,0.048388800024986266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,64,64,128,1,float16,float16,0,0.07080960273742676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,64,64,128,1,float16,float16,0,0.1308832049369812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,64,2,128,1,float16,float16,0,0.026766398549079896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,64,1,128,1,float16,float16,0,0.026849600672721862
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,64,4,128,1,float16,float16,0,0.026892799139022826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,64,1,128,1,float16,float16,0,0.01833920031785965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,64,64,128,1,float16,float16,0,0.037108799815177916
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,64,8,128,1,float16,float16,0,0.028227201104164122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,64,2,128,1,float16,float16,0,0.01820639967918396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,64,4,128,1,float16,float16,0,0.018488000333309173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,64,1,128,1,float16,float16,0,0.015659199655056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,64,8,128,1,float16,float16,0,0.01910240054130554
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,64,64,128,1,float16,float16,0,0.025206398963928223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,64,2,128,1,float16,float16,0,0.015563200414180755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,64,4,128,1,float16,float16,0,0.01565759927034378
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,64,8,128,1,float16,float16,0,0.0161408007144928
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,64,64,128,1,float16,float16,0,0.018617600202560425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,64,4,128,1,float16,float16,0,0.015105600655078887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,64,64,128,1,float16,float16,0,0.016672000288963318
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,64,1,128,1,float16,float16,0,0.015033599734306336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,64,8,128,1,float16,float16,0,0.01555359959602356
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,64,2,128,1,float16,float16,0,0.014974400401115417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,64,4,128,1,float16,float16,0,0.015080000460147857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,64,1,128,1,float16,float16,0,0.015011200308799743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,64,2,128,1,float16,float16,0,0.014945599436759948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,64,8,128,1,float16,float16,0,0.01518079936504364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,64,2,128,1,float16,float16,0,0.12880959510803222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,64,4,128,1,float16,float16,0,0.1364351987838745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,64,8,128,1,float16,float16,0,0.14959679841995238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,64,2,128,1,float16,float16,0,0.07629119753837585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,64,1,128,1,float16,float16,0,0.12733919620513917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,64,1,128,1,float16,float16,0,0.07535840272903442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,64,4,128,1,float16,float16,0,0.0802511990070343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,64,8,128,1,float16,float16,0,0.08583840131759643
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,64,1,128,1,float16,float16,0,0.048028799891471866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,64,64,128,1,float16,float16,0,0.3797616004943848
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,64,64,128,1,float16,float16,0,0.19578239917755128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,64,8,128,1,float16,float16,0,0.053235197067260744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,64,2,128,1,float16,float16,0,0.049563199281692505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,64,4,128,1,float16,float16,0,0.05066239833831787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,64,1,128,1,float16,float16,0,0.027008000016212463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,64,64,128,1,float16,float16,0,0.10583679676055908
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,64,2,128,1,float16,float16,0,0.02738080024719238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,64,4,128,1,float16,float16,0,0.02732959985733032
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,64,2,128,1,float16,float16,0,0.018163199722766876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,64,8,128,1,float16,float16,0,0.028372800350189208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,64,64,128,1,float16,float16,0,0.05358240008354187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,64,4,128,1,float16,float16,0,0.018190400302410127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,64,1,128,1,float16,float16,0,0.01815200001001358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,64,8,128,1,float16,float16,0,0.01908160001039505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,64,1,128,1,float16,float16,0,0.015502400696277618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,64,64,128,1,float16,float16,0,0.03239839971065521
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,64,2,128,1,float16,float16,0,0.01549919992685318
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,64,4,128,1,float16,float16,0,0.015432000160217285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,64,8,128,1,float16,float16,0,0.015932799875736238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,64,1,128,1,float16,float16,0,0.015147200226783753
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,64,2,128,1,float16,float16,0,0.015110400319099427
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,64,64,128,1,float16,float16,0,0.024115200340747833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,64,1,128,1,float16,float16,0,0.014881600439548493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,64,4,128,1,float16,float16,0,0.01515520066022873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,64,64,128,1,float16,float16,0,0.01839679926633835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,64,8,128,1,float16,float16,0,0.015300799906253815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,64,2,128,1,float16,float16,0,0.014859199523925781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,64,4,128,1,float16,float16,0,0.015020799636840821
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,64,8,128,1,float16,float16,0,0.014908799529075622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,64,64,128,1,float16,float16,0,0.01631679981946945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,64,1,128,1,float16,float16,0,0.014609600603580474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,64,2,128,1,float16,float16,0,0.015035200119018554
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,64,4,128,1,float16,float16,0,0.014852799475193024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,64,8,128,1,float16,float16,0,0.015015999972820281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,64,1,128,1,float16,float16,0,0.02757920026779175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,64,2,128,1,float16,float16,0,0.034780800342559814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,64,4,128,1,float16,float16,0,0.051951998472213747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,64,1,128,1,float16,float16,0,0.01919520050287247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,64,2,128,1,float16,float16,0,0.024779200553894043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,64,8,128,1,float16,float16,0,0.08913599848747253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,64,4,128,1,float16,float16,0,0.03229919970035553
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,64,8,128,1,float16,float16,0,0.049860799312591554
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,64,64,128,1,float16,float16,0,0.29501121044158934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,64,1,128,1,float16,float16,0,0.015855999290943147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,64,2,128,1,float16,float16,0,0.017739200592041017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,64,64,128,1,float16,float16,0,0.1530351996421814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,64,4,128,1,float16,float16,0,0.02341119945049286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,64,8,128,1,float16,float16,0,0.030899199843406677
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,64,1,128,1,float16,float16,0,0.015092800557613372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,64,8,128,1,float16,float16,0,0.02282560020685196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,64,2,128,1,float16,float16,0,0.015377600491046906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,64,4,128,1,float16,float16,0,0.017110399901866913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,64,64,128,1,float16,float16,0,0.04761280119419098
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,64,64,128,1,float16,float16,0,0.08411999940872192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,64,1,128,1,float16,float16,0,0.014468799531459808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,64,2,128,1,float16,float16,0,0.01417119950056076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,64,4,128,1,float16,float16,0,0.014443199336528777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,64,8,128,1,float16,float16,0,0.016339200735092162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,64,64,128,1,float16,float16,0,0.03054080009460449
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,64,2,128,1,float16,float16,0,0.01424960047006607
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,64,1,128,1,float16,float16,0,0.014444799721240997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,64,4,128,1,float16,float16,0,0.014235199987888336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,64,8,128,1,float16,float16,0,0.014596800506114959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,64,64,128,1,float16,float16,0,0.02335360050201416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,64,1,128,1,float16,float16,0,0.01417279988527298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,64,2,128,1,float16,float16,0,0.014259199798107147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,64,4,128,1,float16,float16,0,0.014254400134086609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,64,8,128,1,float16,float16,0,0.014369599521160126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,64,64,128,1,float16,float16,0,0.01807039976119995
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,64,1,128,1,float16,float16,0,0.014361600577831268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,64,2,128,1,float16,float16,0,0.014286400377750396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,64,8,128,1,float16,float16,0,0.014103999733924866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,64,4,128,1,float16,float16,0,0.014185599982738495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,64,64,128,1,float16,float16,0,0.015936000645160674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,64,1,128,1,float16,float16,0,0.014192000031471252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,64,4,128,1,float16,float16,0,0.013950400054454803
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,64,2,128,1,float16,float16,0,0.014156800508499146
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,64,8,128,1,float16,float16,0,0.014006400108337402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,48,1,128,1,float16,float16,0,63.84657592773438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,48,2,128,1,float16,float16,0,64.88621215820312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,48,4,128,1,float16,float16,0,63.8794677734375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,48,1,128,1,float16,float16,0,31.981512451171874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,48,1,128,1,float16,float16,0,126.05986328125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,48,8,128,1,float16,float16,0,126.81368408203124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,48,2,128,1,float16,float16,0,128.13695068359374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,48,4,128,1,float16,float16,0,128.259716796875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,48,8,128,1,float16,float16,0,64.43981323242187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,48,2,128,1,float16,float16,0,32.01292724609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,48,1,128,1,float16,float16,0,16.177554321289062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,48,2,128,1,float16,float16,0,16.145411682128906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,48,4,128,1,float16,float16,0,16.14556427001953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,48,4,128,1,float16,float16,0,31.763583374023437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,48,8,128,1,float16,float16,0,31.928375244140625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,48,8,128,1,float16,float16,0,16.3517333984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,48,1,128,1,float16,float16,0,72.18724365234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,48,2,128,1,float16,float16,0,71.98298950195313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,48,4,128,1,float16,float16,0,71.50889282226562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,48,8,128,1,float16,float16,0,71.96430053710938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,48,1,128,1,float16,float16,0,36.18421325683594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,48,2,128,1,float16,float16,0,36.11307373046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,48,4,128,1,float16,float16,0,35.8255859375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,48,8,128,1,float16,float16,0,36.05208129882813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,48,1,128,1,float16,float16,0,18.0525634765625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,48,2,128,1,float16,float16,0,17.976072692871092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,48,4,128,1,float16,float16,0,17.983534240722655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,48,48,128,1,float16,float16,0,16.97392120361328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,48,48,128,1,float16,float16,0,38.374822998046874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,48,1,128,1,float16,float16,0,8.986441802978515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,48,48,128,1,float16,float16,0,33.795040893554685
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,48,48,128,1,float16,float16,0,19.150331115722658
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,48,8,128,1,float16,float16,0,18.375645446777344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,48,2,128,1,float16,float16,0,9.067616271972657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,48,48,128,1,float16,float16,0,9.63572006225586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,48,4,128,1,float16,float16,0,9.198060607910156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,48,8,128,1,float16,float16,0,9.207472229003907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,48,48,128,1,float16,float16,0,65.93878784179688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,48,48,128,1,float16,float16,0,26.565216064453125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,48,1,128,1,float16,float16,0,25.16640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,48,2,128,1,float16,float16,0,24.95397033691406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,48,4,128,1,float16,float16,0,24.991856384277344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,48,1,128,1,float16,float16,0,12.527649688720704
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,48,48,128,1,float16,float16,0,13.666517639160157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,48,1,128,1,float16,float16,0,50.88916015625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,48,2,128,1,float16,float16,0,50.38643188476563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,48,4,128,1,float16,float16,0,50.3682861328125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,48,8,128,1,float16,float16,0,50.93735961914062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,48,8,128,1,float16,float16,0,25.079006958007813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,48,2,128,1,float16,float16,0,12.49432144165039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,48,4,128,1,float16,float16,0,12.674823760986328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,48,1,128,1,float16,float16,0,6.34266242980957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,48,48,128,1,float16,float16,0,6.979342651367188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,48,2,128,1,float16,float16,0,6.430583953857422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,48,4,128,1,float16,float16,0,6.37239990234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,48,8,128,1,float16,float16,0,6.4073020935058596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,48,8,128,1,float16,float16,0,12.914958190917968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,48,2,128,1,float16,float16,0,32.04897766113281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,48,1,128,1,float16,float16,0,32.57618408203125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,48,48,128,1,float16,float16,0,35.07803649902344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,48,4,128,1,float16,float16,0,33.09380187988281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,48,1,128,1,float16,float16,0,16.33411560058594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,48,48,128,1,float16,float16,0,17.63697204589844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,48,2,128,1,float16,float16,0,16.404798889160155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,48,1,128,1,float16,float16,0,63.896051025390626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,48,2,128,1,float16,float16,0,64.02105712890625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,48,48,128,1,float16,float16,0,8.957478332519532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,48,4,128,1,float16,float16,0,64.84564819335938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,48,8,128,1,float16,float16,0,65.4654296875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,48,8,128,1,float16,float16,0,32.28871459960938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,48,4,128,1,float16,float16,0,16.40825500488281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,48,1,128,1,float16,float16,0,8.011366271972657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,48,48,128,1,float16,float16,0,4.517745590209961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,48,8,128,1,float16,float16,0,16.4904052734375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,48,2,128,1,float16,float16,0,8.084731292724609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,48,1,128,1,float16,float16,0,4.079185485839844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,48,4,128,1,float16,float16,0,8.347036743164063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,48,8,128,1,float16,float16,0,8.286094665527344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,48,4,128,1,float16,float16,0,4.09870719909668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,48,2,128,1,float16,float16,0,4.158467102050781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,48,8,128,1,float16,float16,0,4.202660751342774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,48,1,128,1,float16,float16,0,18.120053100585938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,48,2,128,1,float16,float16,0,18.540704345703126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,48,4,128,1,float16,float16,0,18.46173858642578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,48,48,128,1,float16,float16,0,19.99004211425781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,48,48,128,1,float16,float16,0,10.220480346679688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,48,1,128,1,float16,float16,0,9.285145568847657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,48,2,128,1,float16,float16,0,9.219843292236328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,48,1,128,1,float16,float16,0,36.95890502929687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,48,2,128,1,float16,float16,0,36.477377319335936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,48,4,128,1,float16,float16,0,36.34270324707031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,48,8,128,1,float16,float16,0,18.515321350097658
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,48,8,128,1,float16,float16,0,37.697137451171876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,48,4,128,1,float16,float16,0,9.377350616455079
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,48,1,128,1,float16,float16,0,4.620390319824219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,48,2,128,1,float16,float16,0,4.575096130371094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,48,48,128,1,float16,float16,0,5.129919815063476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,48,8,128,1,float16,float16,0,9.488066864013671
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,48,48,128,1,float16,float16,0,2.694532775878906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,48,4,128,1,float16,float16,0,4.737907028198242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,48,1,128,1,float16,float16,0,2.37054557800293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,48,2,128,1,float16,float16,0,2.3754047393798827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,48,4,128,1,float16,float16,0,2.3718799591064452
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,48,8,128,1,float16,float16,0,4.7052257537841795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,48,8,128,1,float16,float16,0,2.487615966796875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,48,1,128,1,float16,float16,0,16.41723175048828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,48,2,128,1,float16,float16,0,16.43513641357422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,48,4,128,1,float16,float16,0,16.734495544433592
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,48,48,128,1,float16,float16,0,19.016629028320313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,48,1,128,1,float16,float16,0,8.309171295166015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,48,48,128,1,float16,float16,0,9.425625610351563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,48,2,128,1,float16,float16,0,8.433611297607422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,48,1,128,1,float16,float16,0,33.26401977539062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,48,2,128,1,float16,float16,0,33.80836181640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,48,4,128,1,float16,float16,0,33.096109008789064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,48,48,128,1,float16,float16,0,4.786438369750977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,48,8,128,1,float16,float16,0,16.8703125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,48,8,128,1,float16,float16,0,34.124365234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,48,4,128,1,float16,float16,0,8.487169647216797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,48,8,128,1,float16,float16,0,8.410478210449218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,48,1,128,1,float16,float16,0,4.186902236938477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,48,1,128,1,float16,float16,0,2.086524772644043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,48,2,128,1,float16,float16,0,2.1153263092041015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,48,2,128,1,float16,float16,0,4.197081756591797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,48,4,128,1,float16,float16,0,4.154336166381836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,48,4,128,1,float16,float16,0,2.128659248352051
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,48,8,128,1,float16,float16,0,4.267057418823242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,48,48,128,1,float16,float16,0,1.2845919609069825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,48,1,128,1,float16,float16,0,1.1015551567077637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,48,8,128,1,float16,float16,0,2.1827119827270507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,48,2,128,1,float16,float16,0,1.108955192565918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,48,8,128,1,float16,float16,0,1.1584287643432618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,48,4,128,1,float16,float16,0,1.1162704467773437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,48,48,128,1,float16,float16,0,2.3377407073974608
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,48,1,128,1,float16,float16,0,9.392662048339844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,48,2,128,1,float16,float16,0,9.437139129638672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,48,4,128,1,float16,float16,0,9.443524932861328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,48,48,128,1,float16,float16,0,11.362792205810546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,48,48,128,1,float16,float16,0,5.580803298950196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,48,1,128,1,float16,float16,0,4.744126510620117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,48,2,128,1,float16,float16,0,4.844459152221679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,48,1,128,1,float16,float16,0,19.182545471191407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,48,2,128,1,float16,float16,0,19.15998229980469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,48,4,128,1,float16,float16,0,19.147119140625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,48,8,128,1,float16,float16,0,19.582020568847657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,48,8,128,1,float16,float16,0,9.634820556640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,48,48,128,1,float16,float16,0,2.8563983917236326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,48,1,128,1,float16,float16,0,2.3935455322265624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,48,4,128,1,float16,float16,0,4.8033710479736325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,48,2,128,1,float16,float16,0,2.44201602935791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,48,4,128,1,float16,float16,0,2.4663360595703123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,48,8,128,1,float16,float16,0,4.823814392089844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,48,48,128,1,float16,float16,0,1.4856767654418945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,48,1,128,1,float16,float16,0,1.2454496383666993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,48,2,128,1,float16,float16,0,1.2476559638977052
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,48,8,128,1,float16,float16,0,2.4459503173828123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,48,4,128,1,float16,float16,0,1.2890031814575196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,48,1,128,1,float16,float16,0,0.6642560005187989
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,48,48,128,1,float16,float16,0,0.7998816013336182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,48,2,128,1,float16,float16,0,0.6706031799316406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,48,4,128,1,float16,float16,0,0.6853328227996827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,48,8,128,1,float16,float16,0,1.282761573791504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,48,8,128,1,float16,float16,0,0.7040304183959961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,48,2,128,1,float16,float16,0,8.692475128173829
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,48,1,128,1,float16,float16,0,8.829319763183594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,48,48,128,1,float16,float16,0,10.885704040527344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,48,4,128,1,float16,float16,0,8.856183624267578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,48,48,128,1,float16,float16,0,5.502840042114258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,48,1,128,1,float16,float16,0,4.308777618408203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,48,1,128,1,float16,float16,0,17.844296264648438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,48,4,128,1,float16,float16,0,17.692979431152345
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,48,2,128,1,float16,float16,0,18.028570556640624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,48,8,128,1,float16,float16,0,18.21257629394531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,48,8,128,1,float16,float16,0,9.053129577636719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,48,2,128,1,float16,float16,0,4.372657775878906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,48,4,128,1,float16,float16,0,4.451526260375976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,48,1,128,1,float16,float16,0,2.154612731933594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,48,2,128,1,float16,float16,0,2.220583915710449
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,48,48,128,1,float16,float16,0,2.790804862976074
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,48,8,128,1,float16,float16,0,4.463343811035156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,48,1,128,1,float16,float16,0,1.1287887573242188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,48,8,128,1,float16,float16,0,2.2188432693481444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,48,48,128,1,float16,float16,0,1.4189295768737793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,48,4,128,1,float16,float16,0,2.2128271102905273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,48,2,128,1,float16,float16,0,1.1198448181152343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,48,48,128,1,float16,float16,0,0.7558447837829589
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,48,4,128,1,float16,float16,0,1.1453968048095704
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,48,8,128,1,float16,float16,0,1.1922112464904786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,48,1,128,1,float16,float16,0,0.5783504009246826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,48,1,128,1,float16,float16,0,0.32389121055603026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,48,2,128,1,float16,float16,0,0.5985455989837647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,48,4,128,1,float16,float16,0,0.6136640071868896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,48,8,128,1,float16,float16,0,0.6293632030487061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,48,48,128,1,float16,float16,0,0.41317119598388674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,48,2,128,1,float16,float16,0,0.3425839900970459
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,48,4,128,1,float16,float16,0,0.356116795539856
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,48,8,128,1,float16,float16,0,0.347321605682373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,48,2,128,1,float16,float16,0,5.116321563720703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,48,1,128,1,float16,float16,0,5.105385589599609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,48,4,128,1,float16,float16,0,5.171030426025391
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,48,48,128,1,float16,float16,0,6.677726745605469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,48,1,128,1,float16,float16,0,2.558401679992676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,48,48,128,1,float16,float16,0,3.405227279663086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,48,2,128,1,float16,float16,0,2.516219139099121
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,48,1,128,1,float16,float16,0,10.064514923095704
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,48,2,128,1,float16,float16,0,10.139068603515625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,48,4,128,1,float16,float16,0,10.322808074951173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,48,8,128,1,float16,float16,0,10.694512176513673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,48,8,128,1,float16,float16,0,5.339115142822266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,48,4,128,1,float16,float16,0,2.586590385437012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,48,1,128,1,float16,float16,0,1.282539176940918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,48,2,128,1,float16,float16,0,1.3076911926269532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,48,4,128,1,float16,float16,0,1.3300559997558594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,48,48,128,1,float16,float16,0,1.677052879333496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,48,48,128,1,float16,float16,0,0.9031472206115723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,48,8,128,1,float16,float16,0,2.673476791381836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,48,1,128,1,float16,float16,0,0.6945456027984619
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,48,2,128,1,float16,float16,0,0.6779471874237061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,48,8,128,1,float16,float16,0,1.362070369720459
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,48,4,128,1,float16,float16,0,0.6949440002441406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,48,48,128,1,float16,float16,0,0.4702064037322998
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,48,1,128,1,float16,float16,0,0.3684783935546875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,48,2,128,1,float16,float16,0,0.38316318988800047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,48,8,128,1,float16,float16,0,0.7172143936157227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,48,4,128,1,float16,float16,0,0.3852672100067139
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,48,48,128,1,float16,float16,0,0.2774399995803833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,48,8,128,1,float16,float16,0,0.3799760103225708
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,48,1,128,1,float16,float16,0,0.20803680419921874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,48,2,128,1,float16,float16,0,0.2036463975906372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,48,8,128,1,float16,float16,0,0.22931039333343506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,48,4,128,1,float16,float16,0,0.2132944107055664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,48,1,128,1,float16,float16,0,4.7603904724121096
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,48,2,128,1,float16,float16,0,4.881171035766601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,48,4,128,1,float16,float16,0,4.9429889678955075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,48,48,128,1,float16,float16,0,6.981646728515625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,48,1,128,1,float16,float16,0,2.3762128829956053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,48,48,128,1,float16,float16,0,3.5356544494628905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,48,1,128,1,float16,float16,0,9.631883239746093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,48,2,128,1,float16,float16,0,2.4082944869995115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,48,2,128,1,float16,float16,0,9.807218933105469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,48,4,128,1,float16,float16,0,9.97643814086914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,48,8,128,1,float16,float16,0,10.137820434570312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,48,8,128,1,float16,float16,0,5.04401741027832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,48,4,128,1,float16,float16,0,2.4882768630981444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,48,1,128,1,float16,float16,0,1.1954031944274903
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,48,48,128,1,float16,float16,0,1.7811904907226563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,48,48,128,1,float16,float16,0,0.9068511962890625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,48,8,128,1,float16,float16,0,2.5625696182250977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,48,2,128,1,float16,float16,0,1.2099184036254882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,48,1,128,1,float16,float16,0,0.6268496036529541
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,48,4,128,1,float16,float16,0,1.2251791954040527
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,48,8,128,1,float16,float16,0,1.2551471710205078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,48,2,128,1,float16,float16,0,0.6208000183105469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,48,48,128,1,float16,float16,0,0.47411041259765624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,48,1,128,1,float16,float16,0,0.3315887928009033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,48,2,128,1,float16,float16,0,0.33498239517211914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,48,4,128,1,float16,float16,0,0.3415888071060181
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,48,4,128,1,float16,float16,0,0.6262432098388672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,48,8,128,1,float16,float16,0,0.6876143932342529
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,48,8,128,1,float16,float16,0,0.3489567995071411
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,48,1,128,1,float16,float16,0,0.18122719526290892
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,48,48,128,1,float16,float16,0,0.26380960941314696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,48,2,128,1,float16,float16,0,0.19255839586257933
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,48,4,128,1,float16,float16,0,0.1963312029838562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,48,8,128,1,float16,float16,0,0.19670560359954833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,48,2,128,1,float16,float16,0,0.11765600442886352
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,48,48,128,1,float16,float16,0,0.15797120332717896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,48,1,128,1,float16,float16,0,0.11227999925613404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,48,4,128,1,float16,float16,0,0.1172111988067627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,48,8,128,1,float16,float16,0,0.121670401096344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,48,1,128,1,float16,float16,0,2.82031192779541
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,48,2,128,1,float16,float16,0,2.8666336059570314
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,48,48,128,1,float16,float16,0,5.048299026489258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,48,1,128,1,float16,float16,0,5.686379241943359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,48,2,128,1,float16,float16,0,5.758046340942383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,48,4,128,1,float16,float16,0,5.911222457885742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,48,4,128,1,float16,float16,0,2.9958879470825197
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,48,8,128,1,float16,float16,0,6.3880462646484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,48,8,128,1,float16,float16,0,3.130401611328125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,48,1,128,1,float16,float16,0,1.4078127861022949
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,48,2,128,1,float16,float16,0,1.4395152091979981
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,48,2,128,1,float16,float16,0,0.7347695827484131
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,48,1,128,1,float16,float16,0,0.7342912197113037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,48,4,128,1,float16,float16,0,1.4714976310729981
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,48,8,128,1,float16,float16,0,1.5604016304016113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,48,48,128,1,float16,float16,0,2.557169532775879
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,48,1,128,1,float16,float16,0,0.37695999145507814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,48,48,128,1,float16,float16,0,1.2642144203186034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,48,4,128,1,float16,float16,0,0.7451871871948242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,48,2,128,1,float16,float16,0,0.38531200885772704
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,48,8,128,1,float16,float16,0,0.7904880046844482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,48,48,128,1,float16,float16,0,0.6714399814605713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,48,4,128,1,float16,float16,0,0.39019360542297366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,48,2,128,1,float16,float16,0,0.206496000289917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,48,4,128,1,float16,float16,0,0.21118559837341308
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,48,48,128,1,float16,float16,0,0.3551664113998413
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,48,1,128,1,float16,float16,0,0.11807839870452881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,48,48,128,1,float16,float16,0,0.19696320295333863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,48,8,128,1,float16,float16,0,0.4066592216491699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,48,1,128,1,float16,float16,0,0.19731839895248413
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,48,8,128,1,float16,float16,0,0.22138559818267822
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,48,2,128,1,float16,float16,0,0.11996639966964721
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,48,4,128,1,float16,float16,0,0.12225279808044434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,48,48,128,1,float16,float16,0,0.1076367974281311
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,48,1,128,1,float16,float16,0,0.07282080054283142
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,48,8,128,1,float16,float16,0,0.12567520141601562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,48,4,128,1,float16,float16,0,0.07490400075912476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,48,2,128,1,float16,float16,0,0.07083839774131775
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,48,48,128,1,float16,float16,0,0.06818400025367737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,48,8,128,1,float16,float16,0,0.08017759919166564
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,48,2,128,1,float16,float16,0,0.047086399793624875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,48,1,128,1,float16,float16,0,0.042033600807189944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,48,4,128,1,float16,float16,0,0.044921600818634035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,48,8,128,1,float16,float16,0,0.042217600345611575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,48,1,128,1,float16,float16,0,0.9219087600708008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,48,2,128,1,float16,float16,0,0.9402511596679688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,48,4,128,1,float16,float16,0,0.982703971862793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,48,1,128,1,float16,float16,0,1.824510383605957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,48,2,128,1,float16,float16,0,1.8768672943115234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,48,4,128,1,float16,float16,0,1.944500732421875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,48,8,128,1,float16,float16,0,2.113995170593262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,48,48,128,1,float16,float16,0,2.1480464935302734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,48,1,128,1,float16,float16,0,0.25001599788665774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,48,1,128,1,float16,float16,0,0.479201602935791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,48,2,128,1,float16,float16,0,0.48641438484191896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,48,8,128,1,float16,float16,0,0.5516240119934082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,48,48,128,1,float16,float16,0,0.5764224052429199
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,48,4,128,1,float16,float16,0,0.5085919857025146
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,48,2,128,1,float16,float16,0,0.25427999496459963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,48,8,128,1,float16,float16,0,1.0546768188476563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,48,48,128,1,float16,float16,0,1.1009712219238281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,48,4,128,1,float16,float16,0,0.266923189163208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,48,1,128,1,float16,float16,0,0.1377616047859192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,48,2,128,1,float16,float16,0,0.1378432035446167
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,48,4,128,1,float16,float16,0,0.14582560062408448
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,48,48,128,1,float16,float16,0,0.30397279262542726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,48,8,128,1,float16,float16,0,0.1541375994682312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,48,1,128,1,float16,float16,0,0.08092479705810547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,48,48,128,1,float16,float16,0,0.16868640184402467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,48,2,128,1,float16,float16,0,0.08194559812545776
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,48,4,128,1,float16,float16,0,0.08487840294837952
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,48,8,128,1,float16,float16,0,0.28547680377960205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,48,2,128,1,float16,float16,0,0.0501151978969574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,48,8,128,1,float16,float16,0,0.08903200030326844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,48,1,128,1,float16,float16,0,0.04816960096359253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,48,48,128,1,float16,float16,0,0.0915232002735138
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,48,48,128,1,float16,float16,0,0.05008159875869751
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,48,8,128,1,float16,float16,0,0.05620959997177124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,48,4,128,1,float16,float16,0,0.05371999740600586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,48,1,128,1,float16,float16,0,0.031707200407981875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,48,4,128,1,float16,float16,0,0.028696000576019287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,48,48,128,1,float16,float16,0,0.027929601073265076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,48,2,128,1,float16,float16,0,0.030425599217414855
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,48,8,128,1,float16,float16,0,0.03237600028514862
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,48,1,128,1,float16,float16,0,0.023622399568557738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,48,4,128,1,float16,float16,0,0.02385440021753311
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,48,2,128,1,float16,float16,0,0.023494400084018707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,48,8,128,1,float16,float16,0,0.023796799778938293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,48,2,128,1,float16,float16,0,0.37424159049987793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,48,1,128,1,float16,float16,0,0.360752010345459
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,48,1,128,1,float16,float16,0,0.7059775829315186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,48,2,128,1,float16,float16,0,0.7290448188781739
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,48,4,128,1,float16,float16,0,0.7773759841918946
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,48,8,128,1,float16,float16,0,0.8730287551879883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,48,4,128,1,float16,float16,0,0.4011824131011963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,48,48,128,1,float16,float16,0,1.051795196533203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,48,1,128,1,float16,float16,0,0.1930464029312134
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,48,8,128,1,float16,float16,0,0.4516416072845459
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,48,2,128,1,float16,float16,0,0.20019838809967042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,48,48,128,1,float16,float16,0,0.5466544151306152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,48,4,128,1,float16,float16,0,0.21663200855255127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,48,1,128,1,float16,float16,0,0.10678399801254272
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,48,8,128,1,float16,float16,0,0.23878560066223145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,48,4,128,1,float16,float16,0,0.11622079610824584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,48,8,128,1,float16,float16,0,0.1302639961242676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,48,48,128,1,float16,float16,0,0.2928528070449829
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,48,2,128,1,float16,float16,0,0.1102560043334961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,48,1,128,1,float16,float16,0,0.06470879912376404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,48,48,128,1,float16,float16,0,0.16031039953231813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,48,2,128,1,float16,float16,0,0.06581439971923828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,48,4,128,1,float16,float16,0,0.06777600049972535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,48,8,128,1,float16,float16,0,0.0739296019077301
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,48,2,128,1,float16,float16,0,0.03794240057468414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,48,48,128,1,float16,float16,0,0.08352800011634827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,48,1,128,1,float16,float16,0,0.036292800307273866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,48,8,128,1,float16,float16,0,0.04382559955120087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,48,1,128,1,float16,float16,0,0.02324160039424896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,48,2,128,1,float16,float16,0,0.021916800737380983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,48,4,128,1,float16,float16,0,0.04727360010147095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,48,48,128,1,float16,float16,0,0.042921599745750424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,48,48,128,1,float16,float16,0,0.022920000553131103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,48,4,128,1,float16,float16,0,0.023515200614929198
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,48,1,128,1,float16,float16,0,0.018367999792099
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,48,8,128,1,float16,float16,0,0.024131199717521666
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,48,2,128,1,float16,float16,0,0.018649600446224213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,48,4,128,1,float16,float16,0,0.01886560022830963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,48,8,128,1,float16,float16,0,0.018880000710487364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,48,48,128,1,float16,float16,0,0.020640000700950623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,48,1,128,1,float16,float16,0,0.017372800409793852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,48,2,128,1,float16,float16,0,0.017723199725151063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,48,4,128,1,float16,float16,0,0.01807679980993271
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,48,8,128,1,float16,float16,0,0.01804320067167282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,48,4,128,1,float16,float16,0,0.39851040840148927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,48,2,128,1,float16,float16,0,0.19271039962768555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,48,1,128,1,float16,float16,0,0.18821920156478883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,48,1,128,1,float16,float16,0,0.35
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,48,2,128,1,float16,float16,0,0.3654799938201904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,48,8,128,1,float16,float16,0,0.4569536209106445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,48,48,128,1,float16,float16,0,0.5400688171386718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,48,1,128,1,float16,float16,0,0.10533440113067627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,48,4,128,1,float16,float16,0,0.21264960765838622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,48,2,128,1,float16,float16,0,0.1066864013671875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,48,8,128,1,float16,float16,0,0.2372351884841919
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,48,4,128,1,float16,float16,0,0.116867196559906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,48,8,128,1,float16,float16,0,0.12976640462875366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,48,48,128,1,float16,float16,0,0.28273279666900636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,48,1,128,1,float16,float16,0,0.061433601379394534
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,48,2,128,1,float16,float16,0,0.06172159910202026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,48,4,128,1,float16,float16,0,0.065447998046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,48,48,128,1,float16,float16,0,0.15305759906768798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,48,8,128,1,float16,float16,0,0.07322880029678344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,48,48,128,1,float16,float16,0,0.0835536003112793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,48,2,128,1,float16,float16,0,0.03222399950027466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,48,8,128,1,float16,float16,0,0.04050880074501038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,48,4,128,1,float16,float16,0,0.03667519986629486
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,48,1,128,1,float16,float16,0,0.03289119899272919
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,48,2,128,1,float16,float16,0,0.019870400428771973
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,48,4,128,1,float16,float16,0,0.02014240026473999
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,48,1,128,1,float16,float16,0,0.0201664000749588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,48,48,128,1,float16,float16,0,0.04423519968986511
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,48,8,128,1,float16,float16,0,0.020635199546813966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,48,48,128,1,float16,float16,0,0.02136639952659607
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,48,1,128,1,float16,float16,0,0.01684480011463165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,48,2,128,1,float16,float16,0,0.016564799845218657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,48,4,128,1,float16,float16,0,0.016835199296474458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,48,8,128,1,float16,float16,0,0.016847999393939973
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,48,1,128,1,float16,float16,0,0.015307199954986573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,48,48,128,1,float16,float16,0,0.01730239987373352
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,48,2,128,1,float16,float16,0,0.01536639928817749
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,48,8,128,1,float16,float16,0,0.015591999888420105
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,48,4,128,1,float16,float16,0,0.015571199357509613
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,48,48,128,1,float16,float16,0,0.01642879992723465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,48,4,128,1,float16,float16,0,0.015094399452209473
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,48,1,128,1,float16,float16,0,0.015118399262428283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,48,8,128,1,float16,float16,0,0.015137599408626556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,48,2,128,1,float16,float16,0,0.015171200037002563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,48,1,128,1,float16,float16,0,0.17721439599990846
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,48,4,128,1,float16,float16,0,0.20258719921112062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,48,1,128,1,float16,float16,0,0.10064959526062012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,48,2,128,1,float16,float16,0,0.18800159692764282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,48,8,128,1,float16,float16,0,0.2497999906539917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,48,2,128,1,float16,float16,0,0.10415199995040894
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,48,48,128,1,float16,float16,0,0.3578736066818237
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,48,4,128,1,float16,float16,0,0.11175520420074463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,48,8,128,1,float16,float16,0,0.13800480365753173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,48,2,128,1,float16,float16,0,0.06264479756355286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,48,1,128,1,float16,float16,0,0.06159359812736511
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,48,48,128,1,float16,float16,0,0.1866544008255005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,48,4,128,1,float16,float16,0,0.06701120138168334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,48,8,128,1,float16,float16,0,0.07792320251464843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,48,1,128,1,float16,float16,0,0.03133119940757752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,48,48,128,1,float16,float16,0,0.09942240118980408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,48,2,128,1,float16,float16,0,0.03257119953632355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,48,8,128,1,float16,float16,0,0.05426080226898193
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,48,48,128,1,float16,float16,0,0.05651519894599914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,48,1,128,1,float16,float16,0,0.01987359970808029
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,48,2,128,1,float16,float16,0,0.019654400646686554
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,48,4,128,1,float16,float16,0,0.019801600277423857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,48,4,128,1,float16,float16,0,0.03576799929141998
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,48,8,128,1,float16,float16,0,0.02542400062084198
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,48,48,128,1,float16,float16,0,0.028628799319267272
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,48,1,128,1,float16,float16,0,0.016201600432395935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,48,4,128,1,float16,float16,0,0.016361600160598753
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,48,2,128,1,float16,float16,0,0.016495999693870545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,48,2,128,1,float16,float16,0,0.015108799934387207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,48,1,128,1,float16,float16,0,0.015307199954986573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,48,48,128,1,float16,float16,0,0.01929759979248047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,48,8,128,1,float16,float16,0,0.018798400461673737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,48,4,128,1,float16,float16,0,0.015348799526691437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,48,8,128,1,float16,float16,0,0.015548799932003022
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,48,48,128,1,float16,float16,0,0.016422399878501893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,48,2,128,1,float16,float16,0,0.014979200065135955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,48,48,128,1,float16,float16,0,0.01600960046052933
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,48,1,128,1,float16,float16,0,0.014924800395965577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,48,4,128,1,float16,float16,0,0.015108799934387207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,48,8,128,1,float16,float16,0,0.01520639955997467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,48,2,128,1,float16,float16,0,0.014900800585746766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,48,8,128,1,float16,float16,0,0.015163199603557586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,48,4,128,1,float16,float16,0,0.014894400537014008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,48,1,128,1,float16,float16,0,0.014737600088119506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,48,1,128,1,float16,float16,0,0.10166560411453247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,48,2,128,1,float16,float16,0,0.10396319627761841
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,48,4,128,1,float16,float16,0,0.1250815987586975
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,48,8,128,1,float16,float16,0,0.13552000522613525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,48,1,128,1,float16,float16,0,0.06333760023117066
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,48,2,128,1,float16,float16,0,0.06303840279579162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,48,4,128,1,float16,float16,0,0.0727728009223938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,48,48,128,1,float16,float16,0,0.2900223970413208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,48,8,128,1,float16,float16,0,0.07714719772338867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,48,1,128,1,float16,float16,0,0.03244960010051727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,48,48,128,1,float16,float16,0,0.1522096037864685
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,48,2,128,1,float16,float16,0,0.03347359895706177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,48,4,128,1,float16,float16,0,0.04238399863243103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,48,8,128,1,float16,float16,0,0.04685760140419006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,48,48,128,1,float16,float16,0,0.08209599852561951
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,48,1,128,1,float16,float16,0,0.020239999890327452
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,48,2,128,1,float16,float16,0,0.02028159946203232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,48,8,128,1,float16,float16,0,0.02598400115966797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,48,4,128,1,float16,float16,0,0.02624480128288269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,48,48,128,1,float16,float16,0,0.0429504007101059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,48,1,128,1,float16,float16,0,0.016212800145149232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,48,2,128,1,float16,float16,0,0.016113600134849547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,48,4,128,1,float16,float16,0,0.018267199397087097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,48,8,128,1,float16,float16,0,0.01812479943037033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,48,48,128,1,float16,float16,0,0.026499199867248534
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,48,1,128,1,float16,float16,0,0.015049600601196289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,48,4,128,1,float16,float16,0,0.015371200442314149
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,48,8,128,1,float16,float16,0,0.015363200008869171
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,48,2,128,1,float16,float16,0,0.015201599895954132
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,48,48,128,1,float16,float16,0,0.018347199261188506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,48,1,128,1,float16,float16,0,0.01496639996767044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,48,4,128,1,float16,float16,0,0.015087999403476715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,48,2,128,1,float16,float16,0,0.015070399641990662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,48,48,128,1,float16,float16,0,0.01616320013999939
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,48,8,128,1,float16,float16,0,0.014900800585746766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,48,4,128,1,float16,float16,0,0.014795200526714325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,48,8,128,1,float16,float16,0,0.014838400483131408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,48,1,128,1,float16,float16,0,0.014667199552059173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,48,2,128,1,float16,float16,0,0.014820800721645355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,48,48,128,1,float16,float16,0,0.01584160029888153
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,48,2,128,1,float16,float16,0,0.014737600088119506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,48,1,128,1,float16,float16,0,0.014644800126552582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,48,4,128,1,float16,float16,0,0.01483680009841919
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,48,8,128,1,float16,float16,0,0.014721600711345673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,48,2,128,1,float16,float16,0,0.03434079885482788
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,48,4,128,1,float16,float16,0,0.05192959904670715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,48,1,128,1,float16,float16,0,0.026862400770187377
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,48,1,128,1,float16,float16,0,0.018727999925613404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,48,2,128,1,float16,float16,0,0.024718399345874786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,48,8,128,1,float16,float16,0,0.08910239934921264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,48,4,128,1,float16,float16,0,0.032304000854492185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,48,48,128,1,float16,float16,0,0.22486720085144044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,48,8,128,1,float16,float16,0,0.050134402513504026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,48,48,128,1,float16,float16,0,0.11937919855117798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,48,2,128,1,float16,float16,0,0.0177279993891716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,48,1,128,1,float16,float16,0,0.015724800527095795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,48,4,128,1,float16,float16,0,0.023398399353027344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,48,8,128,1,float16,float16,0,0.03118560016155243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,48,48,128,1,float16,float16,0,0.0667743980884552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,48,1,128,1,float16,float16,0,0.015217599272727967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,48,2,128,1,float16,float16,0,0.015011200308799743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,48,4,128,1,float16,float16,0,0.016950400173664094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,48,8,128,1,float16,float16,0,0.02279040068387985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,48,1,128,1,float16,float16,0,0.014327999949455262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,48,48,128,1,float16,float16,0,0.03975679874420166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,48,2,128,1,float16,float16,0,0.014289599657058717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,48,8,128,1,float16,float16,0,0.01621119976043701
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,48,4,128,1,float16,float16,0,0.014567999541759491
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,48,48,128,1,float16,float16,0,0.025110399723052977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,48,1,128,1,float16,float16,0,0.014256000518798828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,48,2,128,1,float16,float16,0,0.014267200231552124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,48,4,128,1,float16,float16,0,0.014287999272346497
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,48,8,128,1,float16,float16,0,0.014315199851989747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,48,1,128,1,float16,float16,0,0.013976000249385834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,48,48,128,1,float16,float16,0,0.017851200699806214
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,48,2,128,1,float16,float16,0,0.014027200639247894
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,48,4,128,1,float16,float16,0,0.014361600577831268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,48,8,128,1,float16,float16,0,0.014387199282646179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,48,48,128,1,float16,float16,0,0.015761600434780122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,48,1,128,1,float16,float16,0,0.014286400377750396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,48,4,128,1,float16,float16,0,0.014027200639247894
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,48,8,128,1,float16,float16,0,0.01408800035715103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,48,2,128,1,float16,float16,0,0.013967999815940857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,48,48,128,1,float16,float16,0,0.015347200632095336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,48,2,128,1,float16,float16,0,0.014108799397945404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,48,1,128,1,float16,float16,0,0.013945600390434265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,48,4,128,1,float16,float16,0,0.01406559944152832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,48,8,128,1,float16,float16,0,0.013995200395584106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,40,1,128,1,float16,float16,0,52.62115478515625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,40,2,128,1,float16,float16,0,53.572698974609374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,40,4,128,1,float16,float16,0,53.476739501953126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,40,1,128,1,float16,float16,0,26.6775146484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,40,4,128,1,float16,float16,0,105.28704833984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,40,8,128,1,float16,float16,0,105.91329345703124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,40,8,128,1,float16,float16,0,53.10943603515625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,40,1,128,1,float16,float16,0,106.91767578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,40,2,128,1,float16,float16,0,107.8628662109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,40,2,128,1,float16,float16,0,26.82756652832031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,40,1,128,1,float16,float16,0,13.51766357421875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,40,2,128,1,float16,float16,0,13.693759155273437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,40,4,128,1,float16,float16,0,13.570976257324219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,40,4,128,1,float16,float16,0,26.57764892578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,40,8,128,1,float16,float16,0,26.825323486328124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,40,8,128,1,float16,float16,0,13.768283081054687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,40,1,128,1,float16,float16,0,60.99752197265625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,40,2,128,1,float16,float16,0,60.2388916015625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,40,4,128,1,float16,float16,0,59.633148193359375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,40,8,128,1,float16,float16,0,60.2906982421875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,40,1,128,1,float16,float16,0,30.629397583007812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,40,2,128,1,float16,float16,0,30.3010009765625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,40,4,128,1,float16,float16,0,29.993551635742186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,40,8,128,1,float16,float16,0,30.162109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,40,1,128,1,float16,float16,0,15.146853637695312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,40,2,128,1,float16,float16,0,14.992376708984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,40,4,128,1,float16,float16,0,15.082911682128906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,40,1,128,1,float16,float16,0,7.535614776611328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,40,8,128,1,float16,float16,0,15.385511779785157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,40,2,128,1,float16,float16,0,7.549710083007812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,40,4,128,1,float16,float16,0,7.769926452636719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,40,8,128,1,float16,float16,0,7.6639549255371096
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,40,40,128,1,float16,float16,0,8.047255706787109
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,40,40,128,1,float16,float16,0,28.133917236328124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,40,40,128,1,float16,float16,0,16.121942138671876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,40,40,128,1,float16,float16,0,13.990428161621093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,40,40,128,1,float16,float16,0,55.56893310546875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,40,40,128,1,float16,float16,0,31.987869262695312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,40,1,128,1,float16,float16,0,42.030157470703124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,40,2,128,1,float16,float16,0,41.581991577148436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,40,40,128,1,float16,float16,0,22.867698669433594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,40,4,128,1,float16,float16,0,41.515814208984374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,40,1,128,1,float16,float16,0,20.62952575683594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,40,2,128,1,float16,float16,0,10.448191833496093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,40,1,128,1,float16,float16,0,10.64951171875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,40,40,128,1,float16,float16,0,11.166244506835938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,40,8,128,1,float16,float16,0,42.61814575195312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,40,40,128,1,float16,float16,0,5.83197135925293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,40,2,128,1,float16,float16,0,20.794337463378906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,40,4,128,1,float16,float16,0,21.196322631835937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,40,8,128,1,float16,float16,0,21.323583984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,40,1,128,1,float16,float16,0,5.269489669799805
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,40,4,128,1,float16,float16,0,10.518787384033203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,40,8,128,1,float16,float16,0,10.639228820800781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,40,2,128,1,float16,float16,0,5.328752136230468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,40,4,128,1,float16,float16,0,5.424583816528321
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,40,8,128,1,float16,float16,0,5.454264068603516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,40,1,128,1,float16,float16,0,27.0701416015625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,40,2,128,1,float16,float16,0,27.50968017578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,40,4,128,1,float16,float16,0,27.163040161132812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,40,40,128,1,float16,float16,0,28.7777099609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,40,1,128,1,float16,float16,0,13.50010986328125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,40,2,128,1,float16,float16,0,13.400962829589844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,40,40,128,1,float16,float16,0,14.811705017089844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,40,1,128,1,float16,float16,0,54.017547607421875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,40,2,128,1,float16,float16,0,54.198211669921875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,40,40,128,1,float16,float16,0,7.511558532714844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,40,4,128,1,float16,float16,0,53.71083984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,40,8,128,1,float16,float16,0,54.009759521484376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,40,8,128,1,float16,float16,0,27.59516296386719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,40,4,128,1,float16,float16,0,13.719635009765625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,40,8,128,1,float16,float16,0,13.657278442382813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,40,1,128,1,float16,float16,0,6.830262756347656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,40,2,128,1,float16,float16,0,6.937368011474609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,40,4,128,1,float16,float16,0,6.808171081542969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,40,40,128,1,float16,float16,0,3.8020721435546876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,40,8,128,1,float16,float16,0,6.864766693115234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,40,1,128,1,float16,float16,0,3.4257904052734376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,40,2,128,1,float16,float16,0,3.415009689331055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,40,4,128,1,float16,float16,0,3.496926498413086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,40,8,128,1,float16,float16,0,3.6174911499023437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,40,1,128,1,float16,float16,0,15.32406005859375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,40,2,128,1,float16,float16,0,15.293515014648438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,40,4,128,1,float16,float16,0,15.669343566894531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,40,40,128,1,float16,float16,0,16.651063537597658
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,40,1,128,1,float16,float16,0,7.7865150451660154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,40,2,128,1,float16,float16,0,7.634286499023437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,40,1,128,1,float16,float16,0,30.373141479492187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,40,2,128,1,float16,float16,0,30.923422241210936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,40,40,128,1,float16,float16,0,8.460995483398438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,40,4,128,1,float16,float16,0,30.419488525390626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,40,8,128,1,float16,float16,0,31.037884521484376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,40,8,128,1,float16,float16,0,15.72681884765625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,40,40,128,1,float16,float16,0,4.240683364868164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,40,4,128,1,float16,float16,0,7.903399658203125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,40,8,128,1,float16,float16,0,7.797434997558594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,40,1,128,1,float16,float16,0,3.8631168365478517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,40,2,128,1,float16,float16,0,3.9087696075439453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,40,40,128,1,float16,float16,0,2.2600591659545897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,40,1,128,1,float16,float16,0,1.9683567047119142
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,40,2,128,1,float16,float16,0,2.0231855392456053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,40,4,128,1,float16,float16,0,3.9128032684326173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,40,4,128,1,float16,float16,0,2.0044031143188477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,40,8,128,1,float16,float16,0,2.0867855072021486
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,40,8,128,1,float16,float16,0,4.055350494384766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,40,1,128,1,float16,float16,0,13.666827392578124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,40,2,128,1,float16,float16,0,13.995953369140626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,40,4,128,1,float16,float16,0,13.975303649902344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,40,40,128,1,float16,float16,0,15.646026611328125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,40,1,128,1,float16,float16,0,6.952724456787109
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,40,40,128,1,float16,float16,0,7.964145660400391
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,40,2,128,1,float16,float16,0,6.887267303466797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,40,1,128,1,float16,float16,0,27.818450927734375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,40,2,128,1,float16,float16,0,27.7637451171875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,40,4,128,1,float16,float16,0,27.812472534179687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,40,40,128,1,float16,float16,0,3.9374481201171876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,40,8,128,1,float16,float16,0,14.073634338378906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,40,8,128,1,float16,float16,0,28.88101806640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,40,4,128,1,float16,float16,0,6.982025909423828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,40,8,128,1,float16,float16,0,7.2325439453125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,40,40,128,1,float16,float16,0,2.020568084716797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,40,1,128,1,float16,float16,0,3.4718017578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,40,2,128,1,float16,float16,0,3.5222545623779298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,40,1,128,1,float16,float16,0,1.7822128295898438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,40,2,128,1,float16,float16,0,1.7688159942626953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,40,4,128,1,float16,float16,0,3.505361557006836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,40,8,128,1,float16,float16,0,3.575084686279297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,40,40,128,1,float16,float16,0,1.0896143913269043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,40,1,128,1,float16,float16,0,0.9352368354797364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,40,2,128,1,float16,float16,0,0.9621791839599609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,40,4,128,1,float16,float16,0,1.8425071716308594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,40,4,128,1,float16,float16,0,0.9806591987609863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,40,8,128,1,float16,float16,0,1.0135791778564454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,40,8,128,1,float16,float16,0,1.8337535858154297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,40,1,128,1,float16,float16,0,7.991574096679687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,40,2,128,1,float16,float16,0,7.881049346923828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,40,4,128,1,float16,float16,0,7.955152130126953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,40,40,128,1,float16,float16,0,9.433415985107422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,40,1,128,1,float16,float16,0,3.9300113677978517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,40,40,128,1,float16,float16,0,4.639425659179688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,40,2,128,1,float16,float16,0,4.0307151794433596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,40,1,128,1,float16,float16,0,15.679183959960938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,40,2,128,1,float16,float16,0,16.018357849121095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,40,4,128,1,float16,float16,0,16.172079467773436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,40,40,128,1,float16,float16,0,2.372609519958496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,40,8,128,1,float16,float16,0,8.18582534790039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,40,8,128,1,float16,float16,0,16.74922332763672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,40,4,128,1,float16,float16,0,3.9914417266845703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,40,1,128,1,float16,float16,0,1.9993135452270507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,40,8,128,1,float16,float16,0,4.072201538085937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,40,2,128,1,float16,float16,0,2.002987289428711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,40,1,128,1,float16,float16,0,1.0461183547973634
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,40,2,128,1,float16,float16,0,1.0510031700134277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,40,40,128,1,float16,float16,0,1.235321617126465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,40,4,128,1,float16,float16,0,2.052817535400391
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,40,4,128,1,float16,float16,0,1.0635343551635743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,40,8,128,1,float16,float16,0,1.0944239616394043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,40,1,128,1,float16,float16,0,0.5684319972991944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,40,40,128,1,float16,float16,0,0.6838895797729492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,40,2,128,1,float16,float16,0,0.5792143821716309
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,40,4,128,1,float16,float16,0,0.5910831928253174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,40,8,128,1,float16,float16,0,2.1024911880493162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,40,8,128,1,float16,float16,0,0.6181968212127685
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,40,1,128,1,float16,float16,0,7.243196868896485
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,40,2,128,1,float16,float16,0,7.38671875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,40,4,128,1,float16,float16,0,7.391207885742188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,40,40,128,1,float16,float16,0,9.066606140136718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,40,40,128,1,float16,float16,0,4.603792190551758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,40,1,128,1,float16,float16,0,3.5871246337890623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,40,1,128,1,float16,float16,0,14.4561767578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,40,2,128,1,float16,float16,0,14.628790283203125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,40,4,128,1,float16,float16,0,15.134794616699219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,40,8,128,1,float16,float16,0,7.643720245361328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,40,8,128,1,float16,float16,0,15.439286804199218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,40,2,128,1,float16,float16,0,3.7017585754394533
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,40,40,128,1,float16,float16,0,2.2889856338500976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,40,1,128,1,float16,float16,0,1.8560911178588868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,40,2,128,1,float16,float16,0,1.8626447677612306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,40,4,128,1,float16,float16,0,3.674137496948242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,40,4,128,1,float16,float16,0,1.874577522277832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,40,8,128,1,float16,float16,0,3.771750259399414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,40,2,128,1,float16,float16,0,0.968120002746582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,40,1,128,1,float16,float16,0,0.9013440132141113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,40,40,128,1,float16,float16,0,1.1650511741638183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,40,40,128,1,float16,float16,0,0.6470848083496094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,40,1,128,1,float16,float16,0,0.5208255767822265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,40,4,128,1,float16,float16,0,0.9807375907897949
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,40,8,128,1,float16,float16,0,1.9416463851928711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,40,8,128,1,float16,float16,0,1.0073408126831054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,40,2,128,1,float16,float16,0,0.2938096046447754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,40,2,128,1,float16,float16,0,0.5265552043914795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,40,1,128,1,float16,float16,0,0.2873296022415161
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,40,40,128,1,float16,float16,0,0.3541872024536133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,40,4,128,1,float16,float16,0,0.5199696063995362
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,40,4,128,1,float16,float16,0,0.3130032062530518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,40,8,128,1,float16,float16,0,0.536195182800293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,40,8,128,1,float16,float16,0,0.3131295919418335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,40,1,128,1,float16,float16,0,4.22863998413086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,40,2,128,1,float16,float16,0,4.220977783203125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,40,4,128,1,float16,float16,0,4.265665435791016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,40,40,128,1,float16,float16,0,5.529641723632812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,40,1,128,1,float16,float16,0,2.0866159439086913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,40,40,128,1,float16,float16,0,2.793120002746582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,40,2,128,1,float16,float16,0,2.166908836364746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,40,1,128,1,float16,float16,0,8.465252685546876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,40,2,128,1,float16,float16,0,8.726542663574218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,40,4,128,1,float16,float16,0,8.610431671142578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,40,8,128,1,float16,float16,0,4.5157936096191404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,40,4,128,1,float16,float16,0,2.145577621459961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,40,8,128,1,float16,float16,0,9.049886322021484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,40,1,128,1,float16,float16,0,1.089355182647705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,40,8,128,1,float16,float16,0,2.2073200225830076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,40,40,128,1,float16,float16,0,1.4253071784973144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,40,2,128,1,float16,float16,0,1.1345711708068849
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,40,1,128,1,float16,float16,0,0.5882832050323487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,40,40,128,1,float16,float16,0,0.7464784145355224
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,40,4,128,1,float16,float16,0,1.1196415901184082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,40,2,128,1,float16,float16,0,0.5785840034484864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,40,8,128,1,float16,float16,0,1.1569664001464843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,40,2,128,1,float16,float16,0,0.3156831979751587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,40,1,128,1,float16,float16,0,0.31785919666290285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,40,4,128,1,float16,float16,0,0.32591519355773924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,40,4,128,1,float16,float16,0,0.5873472213745117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,40,40,128,1,float16,float16,0,0.40787358283996583
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,40,8,128,1,float16,float16,0,0.6073696136474609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,40,8,128,1,float16,float16,0,0.3353647947311401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,40,1,128,1,float16,float16,0,0.18729920387268068
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,40,40,128,1,float16,float16,0,0.23708159923553468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,40,2,128,1,float16,float16,0,0.18415199518203734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,40,8,128,1,float16,float16,0,0.19767520427703858
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,40,4,128,1,float16,float16,0,0.18800480365753175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,40,1,128,1,float16,float16,0,3.9872207641601562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,40,2,128,1,float16,float16,0,4.004092788696289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,40,4,128,1,float16,float16,0,4.0588432312011715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,40,40,128,1,float16,float16,0,5.877963256835938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,40,1,128,1,float16,float16,0,1.9733055114746094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,40,40,128,1,float16,float16,0,2.903566360473633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,40,1,128,1,float16,float16,0,8.198069000244141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,40,2,128,1,float16,float16,0,8.21246566772461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,40,4,128,1,float16,float16,0,8.336812591552734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,40,2,128,1,float16,float16,0,2.0236080169677733
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,40,8,128,1,float16,float16,0,8.678982543945313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,40,8,128,1,float16,float16,0,4.379974365234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,40,4,128,1,float16,float16,0,2.028769683837891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,40,1,128,1,float16,float16,0,1.0440064430236817
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,40,4,128,1,float16,float16,0,1.0728495597839356
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,40,40,128,1,float16,float16,0,1.5176095962524414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,40,2,128,1,float16,float16,0,1.040563201904297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,40,1,128,1,float16,float16,0,0.5423888206481934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,40,40,128,1,float16,float16,0,0.7652592182159423
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,40,8,128,1,float16,float16,0,2.1594255447387694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,40,8,128,1,float16,float16,0,1.1070816040039062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,40,2,128,1,float16,float16,0,0.5443999767303467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,40,1,128,1,float16,float16,0,0.28752000331878663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,40,2,128,1,float16,float16,0,0.29375519752502444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,40,40,128,1,float16,float16,0,0.4030288219451904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,40,4,128,1,float16,float16,0,0.3053999900817871
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,40,8,128,1,float16,float16,0,0.30805120468139646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,40,40,128,1,float16,float16,0,0.23136320114135742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,40,8,128,1,float16,float16,0,0.5687488079071045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,40,4,128,1,float16,float16,0,0.549393606185913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,40,1,128,1,float16,float16,0,0.16096960306167601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,40,2,128,1,float16,float16,0,0.1686992049217224
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,40,40,128,1,float16,float16,0,0.14263360500335692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,40,8,128,1,float16,float16,0,0.17950400114059448
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,40,4,128,1,float16,float16,0,0.16922880411148072
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,40,8,128,1,float16,float16,0,0.11112799644470214
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,40,2,128,1,float16,float16,0,0.10327359437942504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,40,1,128,1,float16,float16,0,0.10417920351028442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,40,4,128,1,float16,float16,0,0.10744800567626953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,40,1,128,1,float16,float16,0,2.324331283569336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,40,2,128,1,float16,float16,0,2.370944023132324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,40,1,128,1,float16,float16,0,4.803862380981445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,40,40,128,1,float16,float16,0,4.242033767700195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,40,2,128,1,float16,float16,0,4.815804672241211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,40,4,128,1,float16,float16,0,4.963479995727539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,40,4,128,1,float16,float16,0,2.4964704513549805
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,40,8,128,1,float16,float16,0,5.3847297668457035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,40,8,128,1,float16,float16,0,2.7173040390014647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,40,1,128,1,float16,float16,0,1.2070927619934082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,40,2,128,1,float16,float16,0,1.217091178894043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,40,4,128,1,float16,float16,0,1.2576255798339844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,40,1,128,1,float16,float16,0,0.6260608196258545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,40,2,128,1,float16,float16,0,0.6287007808685303
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,40,40,128,1,float16,float16,0,2.152596855163574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,40,40,128,1,float16,float16,0,1.0992735862731933
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,40,8,128,1,float16,float16,0,1.3791088104248046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,40,1,128,1,float16,float16,0,0.3271647930145264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,40,40,128,1,float16,float16,0,0.57052001953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,40,4,128,1,float16,float16,0,0.6345183849334717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,40,8,128,1,float16,float16,0,0.6893695831298828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,40,2,128,1,float16,float16,0,0.33281280994415285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,40,4,128,1,float16,float16,0,0.3376591920852661
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,40,8,128,1,float16,float16,0,0.37274560928344724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,40,1,128,1,float16,float16,0,0.1762943983078003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,40,40,128,1,float16,float16,0,0.30397601127624513
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,40,2,128,1,float16,float16,0,0.17785120010375977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,40,8,128,1,float16,float16,0,0.1985327959060669
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,40,4,128,1,float16,float16,0,0.18582719564437866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,40,40,128,1,float16,float16,0,0.17044639587402344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,40,2,128,1,float16,float16,0,0.10667200088500976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,40,4,128,1,float16,float16,0,0.11011999845504761
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,40,8,128,1,float16,float16,0,0.11473280191421509
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,40,1,128,1,float16,float16,0,0.1054144024848938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,40,4,128,1,float16,float16,0,0.06291040182113647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,40,1,128,1,float16,float16,0,0.06761919856071472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,40,2,128,1,float16,float16,0,0.061475199460983274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,40,40,128,1,float16,float16,0,0.09815199971199036
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,40,8,128,1,float16,float16,0,0.07337759733200074
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,40,2,128,1,float16,float16,0,0.04472799897193909
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,40,40,128,1,float16,float16,0,0.06228160262107849
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,40,8,128,1,float16,float16,0,0.04373759925365448
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,40,1,128,1,float16,float16,0,0.040303999185562135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,40,4,128,1,float16,float16,0,0.04323680102825165
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,40,2,128,1,float16,float16,0,0.7974271774291992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,40,1,128,1,float16,float16,0,0.7774079799652099
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,40,4,128,1,float16,float16,0,0.8368335723876953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,40,1,128,1,float16,float16,0,1.5347536087036133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,40,2,128,1,float16,float16,0,1.5498656272888183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,40,4,128,1,float16,float16,0,1.6755487442016601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,40,1,128,1,float16,float16,0,0.39961919784545896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,40,40,128,1,float16,float16,0,1.7924543380737306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,40,8,128,1,float16,float16,0,1.893137550354004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,40,2,128,1,float16,float16,0,0.437775993347168
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,40,8,128,1,float16,float16,0,0.9553183555603028
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,40,40,128,1,float16,float16,0,0.9211615562438965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,40,4,128,1,float16,float16,0,0.4485424041748047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,40,8,128,1,float16,float16,0,0.5196928024291992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,40,1,128,1,float16,float16,0,0.218339204788208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,40,40,128,1,float16,float16,0,0.4760928153991699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,40,2,128,1,float16,float16,0,0.2189647912979126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,40,4,128,1,float16,float16,0,0.23893918991088867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,40,40,128,1,float16,float16,0,0.2586303949356079
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,40,8,128,1,float16,float16,0,0.2584768056869507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,40,1,128,1,float16,float16,0,0.11950559616088867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,40,2,128,1,float16,float16,0,0.12569600343704224
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,40,4,128,1,float16,float16,0,0.13032000064849852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,40,1,128,1,float16,float16,0,0.07389919757843018
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,40,40,128,1,float16,float16,0,0.14488639831542968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,40,8,128,1,float16,float16,0,0.1395951986312866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,40,8,128,1,float16,float16,0,0.08251519799232483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,40,4,128,1,float16,float16,0,0.0755840003490448
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,40,40,128,1,float16,float16,0,0.07693600058555602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,40,1,128,1,float16,float16,0,0.04624319970607758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,40,2,128,1,float16,float16,0,0.07293919920921325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,40,8,128,1,float16,float16,0,0.05156639814376831
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,40,2,128,1,float16,float16,0,0.04501279890537262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,40,4,128,1,float16,float16,0,0.04691359996795654
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,40,40,128,1,float16,float16,0,0.04652799963951111
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,40,1,128,1,float16,float16,0,0.02992959916591644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,40,2,128,1,float16,float16,0,0.02975040078163147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,40,1,128,1,float16,float16,0,0.02381120026111603
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,40,4,128,1,float16,float16,0,0.030539199709892273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,40,40,128,1,float16,float16,0,0.026708799600601196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,40,8,128,1,float16,float16,0,0.032523199915885925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,40,2,128,1,float16,float16,0,0.023035199940204622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,40,4,128,1,float16,float16,0,0.023801599442958832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,40,8,128,1,float16,float16,0,0.024531200528144836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,40,1,128,1,float16,float16,0,0.30674879550933837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,40,2,128,1,float16,float16,0,0.3221856117248535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,40,4,128,1,float16,float16,0,0.3498239994049072
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,40,1,128,1,float16,float16,0,0.600278377532959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,40,4,128,1,float16,float16,0,0.6702127933502198
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,40,2,128,1,float16,float16,0,0.6241295814514161
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,40,8,128,1,float16,float16,0,0.7836031913757324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,40,1,128,1,float16,float16,0,0.1658031940460205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,40,2,128,1,float16,float16,0,0.17554399967193604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,40,40,128,1,float16,float16,0,0.8793439865112305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,40,8,128,1,float16,float16,0,0.40889601707458495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,40,4,128,1,float16,float16,0,0.18916319608688353
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,40,8,128,1,float16,float16,0,0.2225183963775635
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,40,1,128,1,float16,float16,0,0.09425280094146729
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,40,40,128,1,float16,float16,0,0.45630722045898436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,40,2,128,1,float16,float16,0,0.09755679965019226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,40,40,128,1,float16,float16,0,0.2464384078979492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,40,4,128,1,float16,float16,0,0.10353280305862426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,40,8,128,1,float16,float16,0,0.12079999446868897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,40,1,128,1,float16,float16,0,0.05700960159301758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,40,40,128,1,float16,float16,0,0.13347519636154176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,40,4,128,1,float16,float16,0,0.06125280261039734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,40,8,128,1,float16,float16,0,0.06911360025405884
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,40,2,128,1,float16,float16,0,0.05889279842376709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,40,1,128,1,float16,float16,0,0.03361920118331909
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,40,4,128,1,float16,float16,0,0.03462879955768585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,40,2,128,1,float16,float16,0,0.034360000491142274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,40,40,128,1,float16,float16,0,0.0707871973514557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,40,40,128,1,float16,float16,0,0.03722400069236755
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,40,8,128,1,float16,float16,0,0.04261119961738587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,40,2,128,1,float16,float16,0,0.023094399273395537
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,40,1,128,1,float16,float16,0,0.022977599501609804
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,40,4,128,1,float16,float16,0,0.023600000143051147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,40,8,128,1,float16,float16,0,0.024214400351047514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,40,40,128,1,float16,float16,0,0.021836799383163453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,40,2,128,1,float16,float16,0,0.018598400056362152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,40,1,128,1,float16,float16,0,0.01802079975605011
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,40,4,128,1,float16,float16,0,0.018534399569034576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,40,1,128,1,float16,float16,0,0.017768000066280366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,40,40,128,1,float16,float16,0,0.020561599731445314
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,40,8,128,1,float16,float16,0,0.019388799369335175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,40,4,128,1,float16,float16,0,0.01810719966888428
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,40,2,128,1,float16,float16,0,0.01780479997396469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,40,8,128,1,float16,float16,0,0.018334400653839112
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,40,1,128,1,float16,float16,0,0.30060639381408694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,40,2,128,1,float16,float16,0,0.3122848033905029
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,40,1,128,1,float16,float16,0,0.16040480136871338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,40,2,128,1,float16,float16,0,0.1684191942214966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,40,4,128,1,float16,float16,0,0.3502799987792969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,40,8,128,1,float16,float16,0,0.41354079246520997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,40,40,128,1,float16,float16,0,0.45278401374816896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,40,4,128,1,float16,float16,0,0.1858896017074585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,40,1,128,1,float16,float16,0,0.09184799790382385
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,40,8,128,1,float16,float16,0,0.21811840534210206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,40,2,128,1,float16,float16,0,0.09587680101394654
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,40,40,128,1,float16,float16,0,0.23932960033416747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,40,4,128,1,float16,float16,0,0.10485919713973998
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,40,1,128,1,float16,float16,0,0.05283359885215759
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,40,40,128,1,float16,float16,0,0.13194080591201782
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,40,4,128,1,float16,float16,0,0.05735840201377869
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,40,2,128,1,float16,float16,0,0.05400639772415161
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,40,8,128,1,float16,float16,0,0.12231040000915527
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,40,8,128,1,float16,float16,0,0.06787999868392944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,40,40,128,1,float16,float16,0,0.06966559886932373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,40,1,128,1,float16,float16,0,0.02984800040721893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,40,2,128,1,float16,float16,0,0.029334399104118346
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,40,4,128,1,float16,float16,0,0.030262398719787597
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,40,8,128,1,float16,float16,0,0.03634720146656036
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,40,40,128,1,float16,float16,0,0.0382095992565155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,40,1,128,1,float16,float16,0,0.018769599497318268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,40,4,128,1,float16,float16,0,0.01899999976158142
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,40,2,128,1,float16,float16,0,0.01863519996404648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,40,8,128,1,float16,float16,0,0.020151999592781068
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,40,1,128,1,float16,float16,0,0.016099199652671814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,40,40,128,1,float16,float16,0,0.020392000675201416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,40,2,128,1,float16,float16,0,0.015863999724388123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,40,8,128,1,float16,float16,0,0.016755199432373045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,40,40,128,1,float16,float16,0,0.016846400499343873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,40,1,128,1,float16,float16,0,0.015166400372982025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,40,4,128,1,float16,float16,0,0.015222400426864624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,40,4,128,1,float16,float16,0,0.01621920019388199
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,40,2,128,1,float16,float16,0,0.015497599542140961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,40,8,128,1,float16,float16,0,0.015659199655056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,40,40,128,1,float16,float16,0,0.016094399988651274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,40,1,128,1,float16,float16,0,0.014985600113868713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,40,2,128,1,float16,float16,0,0.015072000026702882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,40,4,128,1,float16,float16,0,0.014977599680423736
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,40,8,128,1,float16,float16,0,0.015158399939537048
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,40,1,128,1,float16,float16,0,0.15465439558029176
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,40,4,128,1,float16,float16,0,0.18792320489883424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,40,2,128,1,float16,float16,0,0.16311520338058472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,40,1,128,1,float16,float16,0,0.08868160247802734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,40,8,128,1,float16,float16,0,0.23695039749145508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,40,2,128,1,float16,float16,0,0.09342560172080994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,40,4,128,1,float16,float16,0,0.10290399789810181
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,40,8,128,1,float16,float16,0,0.13041119575500487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,40,1,128,1,float16,float16,0,0.05271999835968018
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,40,40,128,1,float16,float16,0,0.15892479419708253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,40,4,128,1,float16,float16,0,0.06155359745025635
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,40,40,128,1,float16,float16,0,0.3030927896499634
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,40,2,128,1,float16,float16,0,0.05424000024795532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,40,1,128,1,float16,float16,0,0.02935200035572052
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,40,8,128,1,float16,float16,0,0.07440159916877746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,40,40,128,1,float16,float16,0,0.08490880131721497
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,40,2,128,1,float16,float16,0,0.0291375994682312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,40,4,128,1,float16,float16,0,0.03083199858665466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,40,40,128,1,float16,float16,0,0.045731198787689206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,40,1,128,1,float16,float16,0,0.01871519982814789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,40,8,128,1,float16,float16,0,0.04247199892997742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,40,8,128,1,float16,float16,0,0.025097599625587462
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,40,40,128,1,float16,float16,0,0.026915198564529418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,40,2,128,1,float16,float16,0,0.018475200235843658
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,40,4,128,1,float16,float16,0,0.019436800479888917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,40,2,128,1,float16,float16,0,0.015787200629711153
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,40,1,128,1,float16,float16,0,0.01579039990901947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,40,4,128,1,float16,float16,0,0.016382400691509248
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,40,8,128,1,float16,float16,0,0.018580800294876097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,40,2,128,1,float16,float16,0,0.014979200065135955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,40,1,128,1,float16,float16,0,0.01510239988565445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,40,4,128,1,float16,float16,0,0.015279999375343323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,40,40,128,1,float16,float16,0,0.01855680048465729
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,40,8,128,1,float16,float16,0,0.015727999806404113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,40,40,128,1,float16,float16,0,0.01602399945259094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,40,1,128,1,float16,float16,0,0.015004800260066986
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,40,2,128,1,float16,float16,0,0.015038399398326874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,40,4,128,1,float16,float16,0,0.015017600357532501
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,40,40,128,1,float16,float16,0,0.015857599675655365
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,40,1,128,1,float16,float16,0,0.014752000570297241
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,40,2,128,1,float16,float16,0,0.014873600006103516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,40,4,128,1,float16,float16,0,0.014689600467681885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,40,8,128,1,float16,float16,0,0.015007999539375306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,40,8,128,1,float16,float16,0,0.015036800503730774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,40,1,128,1,float16,float16,0,0.09047200083732605
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,40,2,128,1,float16,float16,0,0.09666240215301514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,40,4,128,1,float16,float16,0,0.1183519959449768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,40,8,128,1,float16,float16,0,0.127947199344635
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,40,1,128,1,float16,float16,0,0.05398399829864502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,40,2,128,1,float16,float16,0,0.0590224027633667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,40,4,128,1,float16,float16,0,0.06809920072555542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,40,40,128,1,float16,float16,0,0.2458672046661377
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,40,8,128,1,float16,float16,0,0.07342560291290283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,40,2,128,1,float16,float16,0,0.030851200222969055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,40,40,128,1,float16,float16,0,0.12931679487228392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,40,1,128,1,float16,float16,0,0.029627200961112977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,40,4,128,1,float16,float16,0,0.038254401087760924
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,40,40,128,1,float16,float16,0,0.06764000058174133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,40,1,128,1,float16,float16,0,0.01913280040025711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,40,8,128,1,float16,float16,0,0.04154880046844482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,40,2,128,1,float16,float16,0,0.01966720074415207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,40,4,128,1,float16,float16,0,0.0255840003490448
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,40,8,128,1,float16,float16,0,0.025588798522949218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,40,40,128,1,float16,float16,0,0.0351936012506485
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,40,1,128,1,float16,float16,0,0.01587360054254532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,40,2,128,1,float16,float16,0,0.015881599485874177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,40,4,128,1,float16,float16,0,0.018105599284172057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,40,40,128,1,float16,float16,0,0.02563839852809906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,40,8,128,1,float16,float16,0,0.017931200563907623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,40,1,128,1,float16,float16,0,0.015072000026702882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,40,2,128,1,float16,float16,0,0.015230399370193482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,40,4,128,1,float16,float16,0,0.015267199277877808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,40,8,128,1,float16,float16,0,0.01512639969587326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,40,40,128,1,float16,float16,0,0.01770399957895279
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,40,2,128,1,float16,float16,0,0.014820800721645355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,40,1,128,1,float16,float16,0,0.015025599300861359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,40,4,128,1,float16,float16,0,0.015137599408626556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,40,40,128,1,float16,float16,0,0.01564960032701492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,40,8,128,1,float16,float16,0,0.014659200608730317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,40,1,128,1,float16,float16,0,0.014705599844455719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,40,2,128,1,float16,float16,0,0.014711999893188476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,40,8,128,1,float16,float16,0,0.01470080018043518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,40,40,128,1,float16,float16,0,0.01557919979095459
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,40,2,128,1,float16,float16,0,0.014750400185585022
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,40,1,128,1,float16,float16,0,0.014646400511264802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,40,4,128,1,float16,float16,0,0.01481119990348816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,40,4,128,1,float16,float16,0,0.014948800206184387
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,40,8,128,1,float16,float16,0,0.014694400131702423
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,40,1,128,1,float16,float16,0,0.026668798923492432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,40,2,128,1,float16,float16,0,0.034200000762939456
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,40,1,128,1,float16,float16,0,0.018881599605083465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,40,4,128,1,float16,float16,0,0.051820802688598636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,40,8,128,1,float16,float16,0,0.0891439974308014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,40,2,128,1,float16,float16,0,0.024622400104999543
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,40,4,128,1,float16,float16,0,0.03213120102882385
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,40,40,128,1,float16,float16,0,0.18977760076522826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,40,8,128,1,float16,float16,0,0.05003839731216431
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,40,1,128,1,float16,float16,0,0.015779200196266174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,40,2,128,1,float16,float16,0,0.01765599995851517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,40,40,128,1,float16,float16,0,0.10046240091323852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,40,4,128,1,float16,float16,0,0.023260800540447234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,40,1,128,1,float16,float16,0,0.015027199685573579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,40,40,128,1,float16,float16,0,0.05591199994087219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,40,2,128,1,float16,float16,0,0.014876799285411834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,40,8,128,1,float16,float16,0,0.030947199463844298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,40,4,128,1,float16,float16,0,0.016865600645542145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,40,8,128,1,float16,float16,0,0.022598400712013245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,40,40,128,1,float16,float16,0,0.032769599556922914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,40,1,128,1,float16,float16,0,0.014227199554443359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,40,2,128,1,float16,float16,0,0.014395199716091156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,40,4,128,1,float16,float16,0,0.014414399862289429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,40,40,128,1,float16,float16,0,0.024201600253582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,40,8,128,1,float16,float16,0,0.016331200301647187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,40,1,128,1,float16,float16,0,0.0141744002699852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,40,2,128,1,float16,float16,0,0.014251199364662171
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,40,4,128,1,float16,float16,0,0.01419840008020401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,40,8,128,1,float16,float16,0,0.014190399646759033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,40,40,128,1,float16,float16,0,0.01714559942483902
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,40,1,128,1,float16,float16,0,0.014262400567531586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,40,2,128,1,float16,float16,0,0.014176000654697419
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,40,4,128,1,float16,float16,0,0.014215999841690063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,40,8,128,1,float16,float16,0,0.01438560038805008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,40,40,128,1,float16,float16,0,0.015401600301265717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,40,1,128,1,float16,float16,0,0.01383039951324463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,40,2,128,1,float16,float16,0,0.01403679996728897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,40,8,128,1,float16,float16,0,0.014166399836540222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,40,4,128,1,float16,float16,0,0.014177599549293518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,40,1,128,1,float16,float16,0,0.014049600064754485
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,40,40,128,1,float16,float16,0,0.015270400047302245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,40,2,128,1,float16,float16,0,0.014299200475215912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,40,4,128,1,float16,float16,0,0.014144000411033631
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,40,8,128,1,float16,float16,0,0.014124800264835358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,32,1,128,1,float16,float16,0,42.24741821289062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,32,2,128,1,float16,float16,0,42.743804931640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,32,4,128,1,float16,float16,0,43.177023315429686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,32,1,128,1,float16,float16,0,21.518060302734376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,32,1,128,1,float16,float16,0,84.11939697265625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,32,8,128,1,float16,float16,0,84.79026489257812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,32,8,128,1,float16,float16,0,42.676089477539065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,32,4,128,1,float16,float16,0,85.19009399414062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,32,2,128,1,float16,float16,0,86.51260986328126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,32,2,128,1,float16,float16,0,21.71284637451172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,32,1,128,1,float16,float16,0,10.75500030517578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,32,2,128,1,float16,float16,0,10.978662109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,32,4,128,1,float16,float16,0,10.915716552734375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,32,4,128,1,float16,float16,0,21.201504516601563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,32,8,128,1,float16,float16,0,21.43696746826172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,32,8,128,1,float16,float16,0,10.928419494628907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,32,1,128,1,float16,float16,0,48.9349609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,32,2,128,1,float16,float16,0,48.547262573242186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,32,4,128,1,float16,float16,0,47.841207885742186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,32,8,128,1,float16,float16,0,48.17433471679688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,32,1,128,1,float16,float16,0,24.57842254638672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,32,4,128,1,float16,float16,0,23.94855194091797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,32,2,128,1,float16,float16,0,24.386959838867188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,32,8,128,1,float16,float16,0,24.16852264404297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,32,1,128,1,float16,float16,0,12.002371215820313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,32,2,128,1,float16,float16,0,12.205191802978515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,32,4,128,1,float16,float16,0,12.12597427368164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,32,1,128,1,float16,float16,0,6.07214241027832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,32,8,128,1,float16,float16,0,12.16199188232422
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,32,2,128,1,float16,float16,0,6.09874382019043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,32,4,128,1,float16,float16,0,6.187551879882813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,32,8,128,1,float16,float16,0,6.230292892456054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,32,1,128,1,float16,float16,0,33.2120849609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,32,2,128,1,float16,float16,0,33.34060974121094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,32,4,128,1,float16,float16,0,33.483944702148435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,32,8,128,1,float16,float16,0,33.560394287109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,32,32,128,1,float16,float16,0,12.964599609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,32,32,128,1,float16,float16,0,6.5327919006347654
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,32,32,128,1,float16,float16,0,25.329762268066407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,32,32,128,1,float16,float16,0,22.315043640136718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,32,32,128,1,float16,float16,0,8.973302459716797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,32,32,128,1,float16,float16,0,44.71668395996094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,32,1,128,1,float16,float16,0,16.631550598144532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,32,2,128,1,float16,float16,0,16.946640014648438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,32,8,128,1,float16,float16,0,17.113070678710937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,32,4,128,1,float16,float16,0,16.957194519042968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,32,32,128,1,float16,float16,0,11.115271759033202
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,32,32,128,1,float16,float16,0,17.727357482910158
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,32,1,128,1,float16,float16,0,8.407422637939453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,32,1,128,1,float16,float16,0,4.300008010864258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,32,32,128,1,float16,float16,0,4.612809753417968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,32,2,128,1,float16,float16,0,8.485771179199219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,32,4,128,1,float16,float16,0,8.496612548828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,32,8,128,1,float16,float16,0,8.89455337524414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,32,2,128,1,float16,float16,0,4.197798538208008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,32,4,128,1,float16,float16,0,4.302743911743164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,32,8,128,1,float16,float16,0,4.435753631591797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,32,1,128,1,float16,float16,0,21.268191528320312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,32,2,128,1,float16,float16,0,21.52620849609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,32,4,128,1,float16,float16,0,21.799566650390624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,32,32,128,1,float16,float16,0,23.58562774658203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,32,32,128,1,float16,float16,0,11.590660858154298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,32,1,128,1,float16,float16,0,10.792578887939452
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,32,2,128,1,float16,float16,0,11.037454223632812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,32,1,128,1,float16,float16,0,43.308056640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,32,2,128,1,float16,float16,0,43.087274169921876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,32,4,128,1,float16,float16,0,43.484719848632814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,32,32,128,1,float16,float16,0,5.992504119873047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,32,8,128,1,float16,float16,0,43.36298522949219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,32,8,128,1,float16,float16,0,21.622015380859374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,32,4,128,1,float16,float16,0,10.74778060913086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,32,8,128,1,float16,float16,0,11.071022033691406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,32,1,128,1,float16,float16,0,2.7358800888061525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,32,2,128,1,float16,float16,0,5.449545669555664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,32,1,128,1,float16,float16,0,5.482689666748047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,32,4,128,1,float16,float16,0,5.563372802734375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,32,8,128,1,float16,float16,0,5.705416107177735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,32,2,128,1,float16,float16,0,2.7583759307861326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,32,4,128,1,float16,float16,0,2.8197391510009764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,32,8,128,1,float16,float16,0,2.8429216384887694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,32,32,128,1,float16,float16,0,2.978113555908203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,32,32,128,1,float16,float16,0,13.499098205566407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,32,2,128,1,float16,float16,0,12.152500915527344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,32,1,128,1,float16,float16,0,12.253907012939454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,32,4,128,1,float16,float16,0,12.273117065429688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,32,1,128,1,float16,float16,0,6.123892974853516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,32,2,128,1,float16,float16,0,6.097063827514648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,32,32,128,1,float16,float16,0,6.673249816894531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,32,1,128,1,float16,float16,0,24.334103393554688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,32,2,128,1,float16,float16,0,24.752951049804686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,32,4,128,1,float16,float16,0,24.87767639160156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,32,8,128,1,float16,float16,0,25.368794250488282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,32,8,128,1,float16,float16,0,12.438873291015625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,32,4,128,1,float16,float16,0,6.224147033691406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,32,1,128,1,float16,float16,0,3.1131919860839843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,32,32,128,1,float16,float16,0,3.4247905731201174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,32,8,128,1,float16,float16,0,6.277790451049805
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,32,2,128,1,float16,float16,0,3.13896484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,32,4,128,1,float16,float16,0,3.199710464477539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,32,1,128,1,float16,float16,0,1.6022607803344726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,32,32,128,1,float16,float16,0,1.8259872436523437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,32,2,128,1,float16,float16,0,1.6435823440551758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,32,4,128,1,float16,float16,0,1.686769676208496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,32,8,128,1,float16,float16,0,1.7086271286010741
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,32,8,128,1,float16,float16,0,3.194393539428711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,32,1,128,1,float16,float16,0,11.09759979248047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,32,2,128,1,float16,float16,0,11.047353363037109
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,32,4,128,1,float16,float16,0,11.075357055664062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,32,32,128,1,float16,float16,0,12.472676849365234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,32,1,128,1,float16,float16,0,5.434364700317383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,32,2,128,1,float16,float16,0,5.520372772216797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,32,32,128,1,float16,float16,0,6.256417465209961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,32,1,128,1,float16,float16,0,22.39249267578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,32,32,128,1,float16,float16,0,3.185470390319824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,32,2,128,1,float16,float16,0,22.34613494873047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,32,4,128,1,float16,float16,0,22.5650146484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,32,8,128,1,float16,float16,0,22.795916748046874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,32,8,128,1,float16,float16,0,11.466939544677734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,32,4,128,1,float16,float16,0,5.570806503295898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,32,8,128,1,float16,float16,0,5.688606262207031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,32,1,128,1,float16,float16,0,2.8095951080322266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,32,32,128,1,float16,float16,0,1.6587232589721679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,32,1,128,1,float16,float16,0,1.4603327751159667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,32,2,128,1,float16,float16,0,2.808403205871582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,32,4,128,1,float16,float16,0,2.857713508605957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,32,8,128,1,float16,float16,0,2.895025634765625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,32,2,128,1,float16,float16,0,1.4480223655700684
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,32,32,128,1,float16,float16,0,0.8842944145202637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,32,1,128,1,float16,float16,0,0.7684048175811767
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,32,4,128,1,float16,float16,0,1.4818464279174806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,32,2,128,1,float16,float16,0,0.7761023998260498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,32,8,128,1,float16,float16,0,1.5334320068359375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,32,4,128,1,float16,float16,0,0.8327343940734864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,32,8,128,1,float16,float16,0,0.8401776313781738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,32,2,128,1,float16,float16,0,6.441458892822266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,32,1,128,1,float16,float16,0,6.519273376464843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,32,4,128,1,float16,float16,0,6.462462615966797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,32,32,128,1,float16,float16,0,7.382027435302734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,32,1,128,1,float16,float16,0,3.1892463684082033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,32,32,128,1,float16,float16,0,3.82174072265625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,32,2,128,1,float16,float16,0,3.2116031646728516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,32,1,128,1,float16,float16,0,12.608204650878907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,32,2,128,1,float16,float16,0,12.74442367553711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,32,4,128,1,float16,float16,0,12.979118347167969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,32,8,128,1,float16,float16,0,13.076513671875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,32,32,128,1,float16,float16,0,1.9091215133666992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,32,8,128,1,float16,float16,0,6.66173095703125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,32,1,128,1,float16,float16,0,1.5995776176452636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,32,4,128,1,float16,float16,0,3.2344383239746093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,32,2,128,1,float16,float16,0,1.6409135818481446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,32,32,128,1,float16,float16,0,0.9995984077453614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,32,8,128,1,float16,float16,0,3.345159912109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,32,4,128,1,float16,float16,0,1.6538671493530273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,32,1,128,1,float16,float16,0,0.8707072257995605
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,32,2,128,1,float16,float16,0,0.8544256210327148
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,32,8,128,1,float16,float16,0,1.6882463455200196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,32,4,128,1,float16,float16,0,0.8925775527954102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,32,32,128,1,float16,float16,0,0.5620543956756592
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,32,1,128,1,float16,float16,0,0.48605918884277344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,32,4,128,1,float16,float16,0,0.49834718704223635
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,32,2,128,1,float16,float16,0,0.4879199981689453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,32,8,128,1,float16,float16,0,0.8962639808654785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,32,8,128,1,float16,float16,0,0.49841761589050293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,32,1,128,1,float16,float16,0,5.859774398803711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,32,2,128,1,float16,float16,0,5.8405311584472654
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,32,4,128,1,float16,float16,0,5.859022521972657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,32,32,128,1,float16,float16,0,7.3626556396484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,32,1,128,1,float16,float16,0,2.8882816314697264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,32,32,128,1,float16,float16,0,3.6482383728027346
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,32,2,128,1,float16,float16,0,2.947012710571289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,32,1,128,1,float16,float16,0,11.52188491821289
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,32,2,128,1,float16,float16,0,11.806683349609376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,32,4,128,1,float16,float16,0,11.7943359375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,32,8,128,1,float16,float16,0,12.366584014892577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,32,8,128,1,float16,float16,0,6.1165214538574215
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,32,4,128,1,float16,float16,0,2.942323112487793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,32,1,128,1,float16,float16,0,1.4727840423583984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,32,32,128,1,float16,float16,0,1.8473808288574218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,32,2,128,1,float16,float16,0,1.5283535957336425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,32,4,128,1,float16,float16,0,1.5220735549926758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,32,8,128,1,float16,float16,0,3.0532384872436524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,32,1,128,1,float16,float16,0,0.766212797164917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,32,32,128,1,float16,float16,0,0.9854623794555664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,32,2,128,1,float16,float16,0,0.7693327903747559
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,32,1,128,1,float16,float16,0,0.4323584079742432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,32,32,128,1,float16,float16,0,0.5311535835266114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,32,4,128,1,float16,float16,0,0.8180272102355957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,32,8,128,1,float16,float16,0,1.5794495582580566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,32,2,128,1,float16,float16,0,0.43106560707092284
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,32,8,128,1,float16,float16,0,0.8306559562683106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,32,4,128,1,float16,float16,0,0.4443503856658936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,32,1,128,1,float16,float16,0,0.2502239942550659
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,32,2,128,1,float16,float16,0,0.25320959091186523
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,32,8,128,1,float16,float16,0,0.4363840103149414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,32,4,128,1,float16,float16,0,0.24904959201812743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,32,32,128,1,float16,float16,0,0.3011168003082275
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,32,8,128,1,float16,float16,0,0.26601760387420653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,32,1,128,1,float16,float16,0,3.3581615447998048
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,32,2,128,1,float16,float16,0,3.394617462158203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,32,32,128,1,float16,float16,0,4.404211044311523
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,32,1,128,1,float16,float16,0,6.853297424316406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,32,2,128,1,float16,float16,0,6.777854156494141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,32,4,128,1,float16,float16,0,7.035884857177734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,32,32,128,1,float16,float16,0,2.239353561401367
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,32,4,128,1,float16,float16,0,3.4896846771240235
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,32,8,128,1,float16,float16,0,7.126493072509765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,32,8,128,1,float16,float16,0,3.6152896881103516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,32,1,128,1,float16,float16,0,1.7213951110839845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,32,32,128,1,float16,float16,0,1.1495375633239746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,32,1,128,1,float16,float16,0,0.872662353515625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,32,2,128,1,float16,float16,0,0.9087039947509765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,32,2,128,1,float16,float16,0,1.7356832504272461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,32,4,128,1,float16,float16,0,1.7665424346923828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,32,8,128,1,float16,float16,0,1.7605888366699218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,32,1,128,1,float16,float16,0,0.4596240043640137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,32,2,128,1,float16,float16,0,0.4765312194824219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,32,32,128,1,float16,float16,0,0.5983535766601562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,32,1,128,1,float16,float16,0,0.24341120719909667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,32,4,128,1,float16,float16,0,0.9107472419738769
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,32,32,128,1,float16,float16,0,0.33998401165008546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,32,8,128,1,float16,float16,0,0.5157599925994873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,32,4,128,1,float16,float16,0,0.4651440143585205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,32,8,128,1,float16,float16,0,0.9160176277160644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,32,2,128,1,float16,float16,0,0.2717375993728638
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,32,4,128,1,float16,float16,0,0.26835041046142577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,32,1,128,1,float16,float16,0,0.15565760135650636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,32,32,128,1,float16,float16,0,0.19765440225601197
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,32,2,128,1,float16,float16,0,0.17242079973220825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,32,8,128,1,float16,float16,0,0.27295680046081544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,32,4,128,1,float16,float16,0,0.15537919998168945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,32,8,128,1,float16,float16,0,0.17437280416488649
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,32,1,128,1,float16,float16,0,3.162623977661133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,32,2,128,1,float16,float16,0,3.2354896545410154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,32,4,128,1,float16,float16,0,3.276460647583008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,32,32,128,1,float16,float16,0,4.675091171264649
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,32,1,128,1,float16,float16,0,1.615875244140625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,32,1,128,1,float16,float16,0,6.418627166748047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,32,32,128,1,float16,float16,0,2.349188804626465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,32,2,128,1,float16,float16,0,6.453094482421875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,32,4,128,1,float16,float16,0,6.5894287109375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,32,2,128,1,float16,float16,0,1.648993682861328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,32,8,128,1,float16,float16,0,3.4664112091064454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,32,8,128,1,float16,float16,0,7.0794525146484375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,32,4,128,1,float16,float16,0,1.6583471298217773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,32,1,128,1,float16,float16,0,0.8191807746887207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,32,32,128,1,float16,float16,0,1.2008831977844239
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,32,2,128,1,float16,float16,0,0.8364095687866211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,32,4,128,1,float16,float16,0,0.8499872207641601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,32,1,128,1,float16,float16,0,0.4356112003326416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,32,8,128,1,float16,float16,0,1.7493183135986328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,32,2,128,1,float16,float16,0,0.44303197860717775
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,32,32,128,1,float16,float16,0,0.6241871833801269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,32,4,128,1,float16,float16,0,0.4450575828552246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,32,8,128,1,float16,float16,0,0.4687488079071045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,32,8,128,1,float16,float16,0,0.8983296394348145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,32,32,128,1,float16,float16,0,0.33508639335632323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,32,1,128,1,float16,float16,0,0.2399519920349121
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,32,2,128,1,float16,float16,0,0.24746079444885255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,32,8,128,1,float16,float16,0,0.25839359760284425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,32,4,128,1,float16,float16,0,0.24698240756988527
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,32,32,128,1,float16,float16,0,0.1930016040802002
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,32,4,128,1,float16,float16,0,0.1484671950340271
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,32,32,128,1,float16,float16,0,0.11714719533920288
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,32,1,128,1,float16,float16,0,0.13940320014953614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,32,8,128,1,float16,float16,0,0.14898719787597656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,32,2,128,1,float16,float16,0,0.14242559671401978
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,32,2,128,1,float16,float16,0,0.09951519966125488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,32,1,128,1,float16,float16,0,0.09410079717636108
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,32,4,128,1,float16,float16,0,0.09526240229606628
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,32,8,128,1,float16,float16,0,0.09654719829559326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,32,1,128,1,float16,float16,0,1.8552255630493164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,32,2,128,1,float16,float16,0,1.9111856460571288
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,32,4,128,1,float16,float16,0,1.9634000778198242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,32,1,128,1,float16,float16,0,0.9943039894104004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,32,32,128,1,float16,float16,0,3.3432384490966798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,32,1,128,1,float16,float16,0,3.7740863800048827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,32,2,128,1,float16,float16,0,3.8122303009033205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,32,32,128,1,float16,float16,0,1.7137744903564454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,32,4,128,1,float16,float16,0,3.998155212402344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,32,8,128,1,float16,float16,0,2.17346076965332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,32,8,128,1,float16,float16,0,4.359921646118164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,32,1,128,1,float16,float16,0,0.5027167797088623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,32,2,128,1,float16,float16,0,0.991152000427246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,32,4,128,1,float16,float16,0,0.9983839988708496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,32,2,128,1,float16,float16,0,0.5249760150909424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,32,8,128,1,float16,float16,0,1.0957167625427247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,32,32,128,1,float16,float16,0,0.8821087837219238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,32,1,128,1,float16,float16,0,0.26691200733184817
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,32,2,128,1,float16,float16,0,0.26644799709320066
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,32,4,128,1,float16,float16,0,0.5276048183441162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,32,4,128,1,float16,float16,0,0.27845120429992676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,32,8,128,1,float16,float16,0,0.5671040058135987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,32,32,128,1,float16,float16,0,0.2469791889190674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,32,8,128,1,float16,float16,0,0.29861280918121336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,32,32,128,1,float16,float16,0,0.4576416015625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,32,1,128,1,float16,float16,0,0.14726879596710205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,32,4,128,1,float16,float16,0,0.1536064028739929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,32,2,128,1,float16,float16,0,0.15032320022583007
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,32,1,128,1,float16,float16,0,0.08621280193328858
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,32,32,128,1,float16,float16,0,0.14110879898071288
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,32,2,128,1,float16,float16,0,0.09175040125846863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,32,8,128,1,float16,float16,0,0.165555202960968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,32,4,128,1,float16,float16,0,0.09312160015106201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,32,8,128,1,float16,float16,0,0.09919040203094483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,32,32,128,1,float16,float16,0,0.08167679905891419
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,32,1,128,1,float16,float16,0,0.05435360074043274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,32,4,128,1,float16,float16,0,0.05634719729423523
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,32,2,128,1,float16,float16,0,0.052830398082733154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,32,8,128,1,float16,float16,0,0.057811200618743896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,32,32,128,1,float16,float16,0,0.054889601469039914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,32,2,128,1,float16,float16,0,0.040012800693511964
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,32,1,128,1,float16,float16,0,0.03799999952316284
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,32,4,128,1,float16,float16,0,0.04308319985866547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,32,8,128,1,float16,float16,0,0.04326559901237488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,32,1,128,1,float16,float16,0,0.6287983894348145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,32,2,128,1,float16,float16,0,0.6514368057250977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,32,1,128,1,float16,float16,0,1.2309568405151368
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,32,2,128,1,float16,float16,0,1.2828895568847656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,32,4,128,1,float16,float16,0,1.343496036529541
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,32,4,128,1,float16,float16,0,0.6856927871704102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,32,8,128,1,float16,float16,0,1.5204256057739258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,32,32,128,1,float16,float16,0,1.4391632080078125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,32,8,128,1,float16,float16,0,0.7762383937835693
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,32,1,128,1,float16,float16,0,0.33454558849334715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,32,2,128,1,float16,float16,0,0.3487024068832397
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,32,4,128,1,float16,float16,0,0.3584367990493774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,32,32,128,1,float16,float16,0,0.7450704097747802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,32,8,128,1,float16,float16,0,0.41153759956359864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,32,4,128,1,float16,float16,0,0.1920367956161499
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,32,2,128,1,float16,float16,0,0.18322720527648925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,32,8,128,1,float16,float16,0,0.21435999870300293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,32,32,128,1,float16,float16,0,0.38823039531707765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,32,1,128,1,float16,float16,0,0.17675679922103882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,32,32,128,1,float16,float16,0,0.2081216096878052
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,32,2,128,1,float16,float16,0,0.10280159711837769
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,32,4,128,1,float16,float16,0,0.10729279518127441
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,32,8,128,1,float16,float16,0,0.1175663948059082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,32,32,128,1,float16,float16,0,0.11841599941253662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,32,1,128,1,float16,float16,0,0.06238560080528259
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,32,1,128,1,float16,float16,0,0.1024399995803833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,32,4,128,1,float16,float16,0,0.0659280002117157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,32,2,128,1,float16,float16,0,0.06384639739990235
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,32,32,128,1,float16,float16,0,0.06630560159683227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,32,8,128,1,float16,float16,0,0.07131199836730957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,32,1,128,1,float16,float16,0,0.0392192006111145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,32,2,128,1,float16,float16,0,0.03892799913883209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,32,4,128,1,float16,float16,0,0.03952000141143799
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,32,8,128,1,float16,float16,0,0.042630401253700254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,32,32,128,1,float16,float16,0,0.03695679903030395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,32,1,128,1,float16,float16,0,0.028091201186180116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,32,2,128,1,float16,float16,0,0.023038400709629057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,32,1,128,1,float16,float16,0,0.022870400547981264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,32,32,128,1,float16,float16,0,0.027009600400924684
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,32,2,128,1,float16,float16,0,0.029868799448013305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,32,4,128,1,float16,float16,0,0.028708800673484802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,32,8,128,1,float16,float16,0,0.028651198744773863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,32,4,128,1,float16,float16,0,0.023556800186634065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,32,8,128,1,float16,float16,0,0.0236272007226944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,32,1,128,1,float16,float16,0,0.25569920539855956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,32,2,128,1,float16,float16,0,0.2700687885284424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,32,1,128,1,float16,float16,0,0.4875999927520752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,32,2,128,1,float16,float16,0,0.5239712238311768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,32,4,128,1,float16,float16,0,0.5626192092895508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,32,4,128,1,float16,float16,0,0.2976288080215454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,32,8,128,1,float16,float16,0,0.6667247772216797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,32,32,128,1,float16,float16,0,0.7033823966979981
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,32,1,128,1,float16,float16,0,0.14274560213088988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,32,2,128,1,float16,float16,0,0.14776320457458497
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,32,8,128,1,float16,float16,0,0.34803199768066406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,32,4,128,1,float16,float16,0,0.16079039573669435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,32,32,128,1,float16,float16,0,0.37087039947509765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,32,8,128,1,float16,float16,0,0.1924496054649353
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,32,32,128,1,float16,float16,0,0.2019808053970337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,32,1,128,1,float16,float16,0,0.08182719945907593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,32,2,128,1,float16,float16,0,0.08471199870109558
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,32,4,128,1,float16,float16,0,0.09009919762611389
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,32,32,128,1,float16,float16,0,0.11055840253829956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,32,8,128,1,float16,float16,0,0.10595359802246093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,32,4,128,1,float16,float16,0,0.0554639995098114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,32,8,128,1,float16,float16,0,0.05979679822921753
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,32,2,128,1,float16,float16,0,0.05191680192947388
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,32,1,128,1,float16,float16,0,0.05100640058517456
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,32,2,128,1,float16,float16,0,0.029793599247932435
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,32,32,128,1,float16,float16,0,0.058878397941589354
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,32,1,128,1,float16,float16,0,0.030212798714637758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,32,8,128,1,float16,float16,0,0.03237760066986084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,32,4,128,1,float16,float16,0,0.03017599880695343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,32,32,128,1,float16,float16,0,0.02873919904232025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,32,1,128,1,float16,float16,0,0.022734400629997254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,32,2,128,1,float16,float16,0,0.022510400414466857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,32,4,128,1,float16,float16,0,0.021648000180721282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,32,32,128,1,float16,float16,0,0.020926399528980254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,32,8,128,1,float16,float16,0,0.02328319996595383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,32,1,128,1,float16,float16,0,0.01823199987411499
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,32,2,128,1,float16,float16,0,0.018320000171661376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,32,4,128,1,float16,float16,0,0.018812799453735353
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,32,8,128,1,float16,float16,0,0.018534399569034576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,32,32,128,1,float16,float16,0,0.019731199741363524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,32,1,128,1,float16,float16,0,0.017638400197029114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,32,2,128,1,float16,float16,0,0.01772159934043884
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,32,8,128,1,float16,float16,0,0.01791200041770935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,32,4,128,1,float16,float16,0,0.017691199481487275
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,32,1,128,1,float16,float16,0,0.1325376033782959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,32,2,128,1,float16,float16,0,0.13967679738998412
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,32,2,128,1,float16,float16,0,0.2566879987716675
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,32,1,128,1,float16,float16,0,0.2423775911331177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,32,4,128,1,float16,float16,0,0.2873296022415161
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,32,32,128,1,float16,float16,0,0.3703376054763794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,32,8,128,1,float16,float16,0,0.34180479049682616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,32,4,128,1,float16,float16,0,0.1573855996131897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,32,2,128,1,float16,float16,0,0.08105599880218506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,32,1,128,1,float16,float16,0,0.07875840067863464
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,32,8,128,1,float16,float16,0,0.1845247983932495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,32,4,128,1,float16,float16,0,0.09061920046806335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,32,32,128,1,float16,float16,0,0.1950368046760559
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,32,1,128,1,float16,float16,0,0.04833920001983642
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,32,32,128,1,float16,float16,0,0.10829119682312012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,32,2,128,1,float16,float16,0,0.04938080012798309
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,32,8,128,1,float16,float16,0,0.10531040430068969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,32,8,128,1,float16,float16,0,0.058263999223709104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,32,4,128,1,float16,float16,0,0.052035200595855716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,32,1,128,1,float16,float16,0,0.027875199913978577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,32,4,128,1,float16,float16,0,0.027382400631904603
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,32,32,128,1,float16,float16,0,0.058715200424194335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,32,1,128,1,float16,float16,0,0.018651199340820313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,32,2,128,1,float16,float16,0,0.02714560031890869
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,32,32,128,1,float16,float16,0,0.028996801376342772
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,32,8,128,1,float16,float16,0,0.02908959984779358
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,32,4,128,1,float16,float16,0,0.018939200043678283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,32,2,128,1,float16,float16,0,0.018798400461673737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,32,8,128,1,float16,float16,0,0.0192208006978035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,32,32,128,1,float16,float16,0,0.01942880004644394
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,32,1,128,1,float16,float16,0,0.015719999372959138
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,32,2,128,1,float16,float16,0,0.015638400614261628
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,32,4,128,1,float16,float16,0,0.01579360067844391
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,32,8,128,1,float16,float16,0,0.015872000157833098
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,32,1,128,1,float16,float16,0,0.015060800313949584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,32,2,128,1,float16,float16,0,0.015172800421714783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,32,32,128,1,float16,float16,0,0.016491200029850005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,32,8,128,1,float16,float16,0,0.015478399395942689
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,32,4,128,1,float16,float16,0,0.015428799390792846
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,32,32,128,1,float16,float16,0,0.015561600029468537
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,32,1,128,1,float16,float16,0,0.014924800395965577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,32,4,128,1,float16,float16,0,0.01510239988565445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,32,2,128,1,float16,float16,0,0.015137599408626556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,32,8,128,1,float16,float16,0,0.015278400480747223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,32,1,128,1,float16,float16,0,0.1296447992324829
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,32,2,128,1,float16,float16,0,0.13902080059051514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,32,1,128,1,float16,float16,0,0.07628160119056701
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,32,4,128,1,float16,float16,0,0.1504320025444031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,32,8,128,1,float16,float16,0,0.17301119565963746
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,32,4,128,1,float16,float16,0,0.08719840049743652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,32,32,128,1,float16,float16,0,0.24610719680786133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,32,2,128,1,float16,float16,0,0.08086079955101014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,32,8,128,1,float16,float16,0,0.09953920245170593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,32,1,128,1,float16,float16,0,0.04843840003013611
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,32,4,128,1,float16,float16,0,0.05297759771347046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,32,2,128,1,float16,float16,0,0.0494623988866806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,32,32,128,1,float16,float16,0,0.1304751992225647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,32,8,128,1,float16,float16,0,0.057652801275253296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,32,1,128,1,float16,float16,0,0.027489599585533143
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,32,2,128,1,float16,float16,0,0.027422401309013366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,32,32,128,1,float16,float16,0,0.07276480197906494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,32,4,128,1,float16,float16,0,0.02768160104751587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,32,8,128,1,float16,float16,0,0.02967199981212616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,32,4,128,1,float16,float16,0,0.018760000169277192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,32,32,128,1,float16,float16,0,0.03638240098953247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,32,2,128,1,float16,float16,0,0.018529599905014037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,32,1,128,1,float16,float16,0,0.01855199933052063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,32,8,128,1,float16,float16,0,0.019096000492572783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,32,1,128,1,float16,float16,0,0.015398399531841278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,32,32,128,1,float16,float16,0,0.02518239915370941
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,32,2,128,1,float16,float16,0,0.015408000349998474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,32,8,128,1,float16,float16,0,0.016123199462890626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,32,32,128,1,float16,float16,0,0.01812639981508255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,32,4,128,1,float16,float16,0,0.015715199708938598
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,32,1,128,1,float16,float16,0,0.015108799934387207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,32,4,128,1,float16,float16,0,0.015211200714111328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,32,2,128,1,float16,float16,0,0.014974400401115417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,32,8,128,1,float16,float16,0,0.015396800637245179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,32,4,128,1,float16,float16,0,0.015163199603557586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,32,32,128,1,float16,float16,0,0.01584160029888153
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,32,2,128,1,float16,float16,0,0.0149167999625206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,32,8,128,1,float16,float16,0,0.014905600249767304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,32,1,128,1,float16,float16,0,0.01499360054731369
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,32,32,128,1,float16,float16,0,0.01541599929332733
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,32,1,128,1,float16,float16,0,0.014587199687957764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,32,2,128,1,float16,float16,0,0.014739200472831726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,32,8,128,1,float16,float16,0,0.014920000731945039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,32,4,128,1,float16,float16,0,0.014951999485492706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,32,1,128,1,float16,float16,0,0.07739199995994568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,32,2,128,1,float16,float16,0,0.07885919809341431
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,32,1,128,1,float16,float16,0,0.049859198927879336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,32,4,128,1,float16,float16,0,0.08786399960517884
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,32,8,128,1,float16,float16,0,0.12393440008163452
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,32,2,128,1,float16,float16,0,0.05113919973373413
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,32,4,128,1,float16,float16,0,0.05404000282287598
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,32,32,128,1,float16,float16,0,0.19964799880981446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,32,1,128,1,float16,float16,0,0.027692800760269164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,32,8,128,1,float16,float16,0,0.06986719965934754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,32,32,128,1,float16,float16,0,0.10675040483474732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,32,2,128,1,float16,float16,0,0.02781279981136322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,32,4,128,1,float16,float16,0,0.028644800186157227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,32,8,128,1,float16,float16,0,0.03608480095863342
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,32,1,128,1,float16,float16,0,0.01889439970254898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,32,32,128,1,float16,float16,0,0.05335519909858703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,32,4,128,1,float16,float16,0,0.019215999543666838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,32,8,128,1,float16,float16,0,0.024864000082015992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,32,2,128,1,float16,float16,0,0.018889600038528444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,32,32,128,1,float16,float16,0,0.03259359896183014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,32,1,128,1,float16,float16,0,0.01568000018596649
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,32,2,128,1,float16,float16,0,0.015371200442314149
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,32,4,128,1,float16,float16,0,0.015512000024318694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,32,8,128,1,float16,float16,0,0.01764959990978241
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,32,32,128,1,float16,float16,0,0.023713600635528565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,32,2,128,1,float16,float16,0,0.015238399803638458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,32,1,128,1,float16,float16,0,0.014844800531864166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,32,4,128,1,float16,float16,0,0.015225599706172942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,32,8,128,1,float16,float16,0,0.015054400265216827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,32,32,128,1,float16,float16,0,0.017566399276256563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,32,1,128,1,float16,float16,0,0.01481119990348816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,32,4,128,1,float16,float16,0,0.015081599354743958
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,32,2,128,1,float16,float16,0,0.014975999295711518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,32,8,128,1,float16,float16,0,0.014636799693107605
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,32,32,128,1,float16,float16,0,0.015382400155067444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,32,1,128,1,float16,float16,0,0.01465120017528534
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,32,2,128,1,float16,float16,0,0.014727999269962311
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,32,8,128,1,float16,float16,0,0.014590400457382201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,32,4,128,1,float16,float16,0,0.014902399480342865
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,32,32,128,1,float16,float16,0,0.015320000052452088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,32,1,128,1,float16,float16,0,0.014659200608730317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,32,2,128,1,float16,float16,0,0.014854399859905243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,32,4,128,1,float16,float16,0,0.014828799664974213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,32,8,128,1,float16,float16,0,0.014646400511264802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,32,1,128,1,float16,float16,0,0.026662400364875792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,32,4,128,1,float16,float16,0,0.05156480073928833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,32,2,128,1,float16,float16,0,0.03402400016784668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,32,1,128,1,float16,float16,0,0.018825599551200868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,32,8,128,1,float16,float16,0,0.08890079855918884
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,32,32,128,1,float16,float16,0,0.15544639825820922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,32,2,128,1,float16,float16,0,0.02449280023574829
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,32,4,128,1,float16,float16,0,0.03213280141353607
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,32,8,128,1,float16,float16,0,0.04972479939460754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,32,1,128,1,float16,float16,0,0.015729600191116334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,32,32,128,1,float16,float16,0,0.08519999980926514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,32,2,128,1,float16,float16,0,0.017499199509620665
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,32,8,128,1,float16,float16,0,0.03091999888420105
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,32,4,128,1,float16,float16,0,0.02346239984035492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,32,1,128,1,float16,float16,0,0.014927999675273895
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,32,32,128,1,float16,float16,0,0.04812000095844269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,32,2,128,1,float16,float16,0,0.014958399534225463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,32,4,128,1,float16,float16,0,0.016945600509643555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,32,8,128,1,float16,float16,0,0.022702400386333466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,32,1,128,1,float16,float16,0,0.014057600498199463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,32,32,128,1,float16,float16,0,0.030324798822402955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,32,2,128,1,float16,float16,0,0.0143327996134758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,32,8,128,1,float16,float16,0,0.01634880006313324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,32,4,128,1,float16,float16,0,0.014441600441932679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,32,32,128,1,float16,float16,0,0.02252320051193237
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,32,1,128,1,float16,float16,0,0.014151999354362487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,32,2,128,1,float16,float16,0,0.014083200693130493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,32,4,128,1,float16,float16,0,0.014209599792957306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,32,8,128,1,float16,float16,0,0.014339199662208557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,32,1,128,1,float16,float16,0,0.014169600605964661
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,32,32,128,1,float16,float16,0,0.017052799463272095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,32,4,128,1,float16,float16,0,0.01395999938249588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,32,2,128,1,float16,float16,0,0.01414719969034195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,32,8,128,1,float16,float16,0,0.014047999680042268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,32,32,128,1,float16,float16,0,0.015296000242233276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,32,1,128,1,float16,float16,0,0.014113600552082061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,32,2,128,1,float16,float16,0,0.014027200639247894
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,32,4,128,1,float16,float16,0,0.014017599821090698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,32,1,128,1,float16,float16,0,0.014044800400733947
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,32,8,128,1,float16,float16,0,0.01422560065984726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,32,32,128,1,float16,float16,0,0.014790399372577668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,32,4,128,1,float16,float16,0,0.013910399377346038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,32,8,128,1,float16,float16,0,0.014134399592876434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,32,2,128,1,float16,float16,0,0.01388320028781891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,24,1,128,1,float16,float16,0,32.001730346679686
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,24,2,128,1,float16,float16,0,32.071929931640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,24,4,128,1,float16,float16,0,31.772259521484376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,24,1,128,1,float16,float16,0,15.934066772460938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,24,2,128,1,float16,float16,0,63.18157958984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,24,1,128,1,float16,float16,0,63.790228271484374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,24,8,128,1,float16,float16,0,64.56114501953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,24,4,128,1,float16,float16,0,64.90764770507812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,24,8,128,1,float16,float16,0,32.62134704589844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,24,2,128,1,float16,float16,0,15.872361755371093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,24,1,128,1,float16,float16,0,8.250414276123047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,24,2,128,1,float16,float16,0,8.247161865234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,24,4,128,1,float16,float16,0,16.14550018310547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,24,8,128,1,float16,float16,0,16.54918670654297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,24,4,128,1,float16,float16,0,8.38523178100586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,24,8,128,1,float16,float16,0,8.497523498535156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,24,1,128,1,float16,float16,0,35.81578063964844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,24,4,128,1,float16,float16,0,35.948883056640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,24,2,128,1,float16,float16,0,36.259133911132814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,24,8,128,1,float16,float16,0,37.27018737792969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,24,1,128,1,float16,float16,0,17.920294189453124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,24,2,128,1,float16,float16,0,17.98515167236328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,24,4,128,1,float16,float16,0,18.336204528808594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,24,8,128,1,float16,float16,0,18.735763549804688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,24,1,128,1,float16,float16,0,9.012433624267578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,24,2,128,1,float16,float16,0,9.151000213623046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,24,4,128,1,float16,float16,0,9.37560806274414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,24,1,128,1,float16,float16,0,4.694035339355469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,24,8,128,1,float16,float16,0,9.328158569335937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,24,2,128,1,float16,float16,0,4.7588542938232425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,24,4,128,1,float16,float16,0,4.680769729614258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,24,8,128,1,float16,float16,0,4.895431900024414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,24,1,128,1,float16,float16,0,24.98363800048828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,24,2,128,1,float16,float16,0,25.682977294921876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,24,4,128,1,float16,float16,0,25.172279357910156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,24,8,128,1,float16,float16,0,26.106207275390624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,24,1,128,1,float16,float16,0,12.780225372314453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,24,2,128,1,float16,float16,0,12.879853820800781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,24,4,128,1,float16,float16,0,13.079972839355468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,24,8,128,1,float16,float16,0,13.163485717773437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,24,24,128,1,float16,float16,0,19.11157989501953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,24,24,128,1,float16,float16,0,13.284745788574218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,24,24,128,1,float16,float16,0,4.9297935485839846
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,24,1,128,1,float16,float16,0,6.295766448974609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,24,2,128,1,float16,float16,0,6.3301647186279295
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,24,4,128,1,float16,float16,0,6.404148864746094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,24,24,128,1,float16,float16,0,16.990411376953126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,24,24,128,1,float16,float16,0,32.82897033691406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,24,24,128,1,float16,float16,0,9.544140625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,24,24,128,1,float16,float16,0,3.502531051635742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,24,1,128,1,float16,float16,0,3.2510448455810548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,24,2,128,1,float16,float16,0,3.278148651123047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,24,4,128,1,float16,float16,0,3.3126224517822265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,24,8,128,1,float16,float16,0,3.4453392028808594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,24,8,128,1,float16,float16,0,6.600399780273437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,24,24,128,1,float16,float16,0,8.63114242553711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,24,24,128,1,float16,float16,0,6.901420593261719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,24,1,128,1,float16,float16,0,16.238861083984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,24,24,128,1,float16,float16,0,17.244277954101562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,24,2,128,1,float16,float16,0,16.11283721923828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,24,4,128,1,float16,float16,0,16.580479431152344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,24,24,128,1,float16,float16,0,8.72310562133789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,24,1,128,1,float16,float16,0,8.099612426757812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,24,1,128,1,float16,float16,0,31.953924560546874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,24,2,128,1,float16,float16,0,32.42066650390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,24,8,128,1,float16,float16,0,32.63460693359375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,24,4,128,1,float16,float16,0,32.642697143554685
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,24,8,128,1,float16,float16,0,16.750595092773438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,24,24,128,1,float16,float16,0,4.489432144165039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,24,2,128,1,float16,float16,0,8.25543670654297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,24,4,128,1,float16,float16,0,8.200243377685547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,24,1,128,1,float16,float16,0,4.127630233764648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,24,8,128,1,float16,float16,0,8.384677124023437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,24,2,128,1,float16,float16,0,4.077088165283203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,24,4,128,1,float16,float16,0,4.289796829223633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,24,24,128,1,float16,float16,0,2.324115180969238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,24,1,128,1,float16,float16,0,2.1440048217773438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,24,2,128,1,float16,float16,0,2.2193599700927735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,24,4,128,1,float16,float16,0,2.2145599365234374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,24,8,128,1,float16,float16,0,4.305327987670898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,24,8,128,1,float16,float16,0,2.2553936004638673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,24,1,128,1,float16,float16,0,9.243382263183594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,24,2,128,1,float16,float16,0,9.274066925048828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,24,24,128,1,float16,float16,0,10.024937438964844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,24,4,128,1,float16,float16,0,9.373355102539062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,24,1,128,1,float16,float16,0,4.579670333862305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,24,2,128,1,float16,float16,0,4.628099060058593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,24,24,128,1,float16,float16,0,5.119873428344727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,24,1,128,1,float16,float16,0,18.06928405761719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,24,2,128,1,float16,float16,0,18.467015075683594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,24,4,128,1,float16,float16,0,18.428684997558594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,24,24,128,1,float16,float16,0,2.630423927307129
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,24,8,128,1,float16,float16,0,19.258595275878907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,24,8,128,1,float16,float16,0,9.517443084716797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,24,1,128,1,float16,float16,0,2.3147504806518553
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,24,4,128,1,float16,float16,0,4.666870498657227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,24,2,128,1,float16,float16,0,2.4100175857543946
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,24,8,128,1,float16,float16,0,4.787879943847656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,24,4,128,1,float16,float16,0,2.4342863082885744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,24,1,128,1,float16,float16,0,1.2687040328979493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,24,24,128,1,float16,float16,0,1.44857759475708
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,24,2,128,1,float16,float16,0,1.253758430480957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,24,8,128,1,float16,float16,0,2.4909759521484376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,24,4,128,1,float16,float16,0,1.3258671760559082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,24,8,128,1,float16,float16,0,1.3333807945251466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,24,1,128,1,float16,float16,0,8.366264343261719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,24,2,128,1,float16,float16,0,8.308380889892579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,24,4,128,1,float16,float16,0,8.533573150634766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,24,24,128,1,float16,float16,0,9.340631866455078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,24,1,128,1,float16,float16,0,4.231670379638672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,24,2,128,1,float16,float16,0,4.167889785766602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,24,24,128,1,float16,float16,0,4.726631927490234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,24,1,128,1,float16,float16,0,16.609576416015624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,24,2,128,1,float16,float16,0,16.635406494140625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,24,4,128,1,float16,float16,0,17.053663635253905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,24,8,128,1,float16,float16,0,17.578227233886718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,24,8,128,1,float16,float16,0,8.811366271972656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,24,4,128,1,float16,float16,0,4.1976463317871096
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,24,8,128,1,float16,float16,0,4.339688110351562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,24,1,128,1,float16,float16,0,2.1037391662597655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,24,24,128,1,float16,float16,0,2.4076223373413086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,24,2,128,1,float16,float16,0,2.135935974121094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,24,24,128,1,float16,float16,0,1.2743359565734864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,24,1,128,1,float16,float16,0,1.1169039726257324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,24,2,128,1,float16,float16,0,1.1304944038391114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,24,4,128,1,float16,float16,0,2.189905548095703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,24,8,128,1,float16,float16,0,2.229876708984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,24,4,128,1,float16,float16,0,1.1584176063537597
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,24,8,128,1,float16,float16,0,1.178441619873047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,24,1,128,1,float16,float16,0,0.609606409072876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,24,24,128,1,float16,float16,0,0.7083824157714844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,24,2,128,1,float16,float16,0,0.6191887855529785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,24,4,128,1,float16,float16,0,0.667790412902832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,24,8,128,1,float16,float16,0,0.6731520175933838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,24,2,128,1,float16,float16,0,4.801052856445312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,24,1,128,1,float16,float16,0,4.846382522583008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,24,24,128,1,float16,float16,0,5.562343978881836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,24,4,128,1,float16,float16,0,4.834822463989258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,24,24,128,1,float16,float16,0,2.876905632019043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,24,1,128,1,float16,float16,0,2.3768320083618164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,24,1,128,1,float16,float16,0,9.526950073242187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,24,2,128,1,float16,float16,0,9.615590667724609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,24,4,128,1,float16,float16,0,9.628781127929688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,24,2,128,1,float16,float16,0,2.505710411071777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,24,8,128,1,float16,float16,0,9.874803161621093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,24,8,128,1,float16,float16,0,5.063927841186524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,24,1,128,1,float16,float16,0,1.2733167648315429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,24,24,128,1,float16,float16,0,1.4523247718811034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,24,4,128,1,float16,float16,0,2.5078319549560546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,24,2,128,1,float16,float16,0,1.262822437286377
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,24,8,128,1,float16,float16,0,2.5092447280883787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,24,4,128,1,float16,float16,0,1.279751968383789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,24,8,128,1,float16,float16,0,1.3172191619873046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,24,24,128,1,float16,float16,0,0.8102879524230957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,24,1,128,1,float16,float16,0,0.6739488124847413
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,24,2,128,1,float16,float16,0,0.669596815109253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,24,24,128,1,float16,float16,0,0.4552192211151123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,24,1,128,1,float16,float16,0,0.3805023908615112
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,24,4,128,1,float16,float16,0,0.7281199932098389
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,24,2,128,1,float16,float16,0,0.3806096076965332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,24,4,128,1,float16,float16,0,0.41243839263916016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,24,8,128,1,float16,float16,0,0.7141327857971191
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,24,8,128,1,float16,float16,0,0.41251840591430666
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,24,1,128,1,float16,float16,0,4.376736068725586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,24,2,128,1,float16,float16,0,4.367868804931641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,24,4,128,1,float16,float16,0,4.490345764160156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,24,24,128,1,float16,float16,0,5.388067245483398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,24,1,128,1,float16,float16,0,2.204732894897461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,24,24,128,1,float16,float16,0,2.72283992767334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,24,2,128,1,float16,float16,0,2.1764320373535155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,24,1,128,1,float16,float16,0,8.767447662353515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,24,2,128,1,float16,float16,0,8.773331451416016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,24,4,128,1,float16,float16,0,9.051598358154298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,24,8,128,1,float16,float16,0,4.746148681640625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,24,8,128,1,float16,float16,0,9.585842895507813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,24,4,128,1,float16,float16,0,2.2534320831298826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,24,24,128,1,float16,float16,0,1.4000911712646484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,24,1,128,1,float16,float16,0,1.1225232124328612
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,24,8,128,1,float16,float16,0,2.330739212036133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,24,2,128,1,float16,float16,0,1.1475248336791992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,24,4,128,1,float16,float16,0,1.1890912055969238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,24,1,128,1,float16,float16,0,0.6016143798828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,24,2,128,1,float16,float16,0,0.6112336158752442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,24,4,128,1,float16,float16,0,0.6141808032989502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,24,24,128,1,float16,float16,0,0.7522128105163575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,24,24,128,1,float16,float16,0,0.4156991958618164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,24,8,128,1,float16,float16,0,1.2380399703979492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,24,8,128,1,float16,float16,0,0.6521071910858154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,24,2,128,1,float16,float16,0,0.34614560604095457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,24,1,128,1,float16,float16,0,0.33943839073181153
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,24,1,128,1,float16,float16,0,0.19899040460586548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,24,4,128,1,float16,float16,0,0.345961594581604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,24,4,128,1,float16,float16,0,0.2096031904220581
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,24,24,128,1,float16,float16,0,0.2477952003479004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,24,8,128,1,float16,float16,0,0.35605759620666505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,24,2,128,1,float16,float16,0,0.2190016031265259
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,24,8,128,1,float16,float16,0,0.21807360649108887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,24,1,128,1,float16,float16,0,2.473579216003418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,24,2,128,1,float16,float16,0,2.5286256790161135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,24,4,128,1,float16,float16,0,2.569563293457031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,24,24,128,1,float16,float16,0,3.3342784881591796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,24,1,128,1,float16,float16,0,1.3220159530639648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,24,24,128,1,float16,float16,0,1.6998224258422852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,24,1,128,1,float16,float16,0,5.10650863647461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,24,2,128,1,float16,float16,0,1.3457648277282714
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,24,2,128,1,float16,float16,0,5.070320129394531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,24,4,128,1,float16,float16,0,5.1821441650390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,24,8,128,1,float16,float16,0,2.7931903839111327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,24,8,128,1,float16,float16,0,5.517464065551758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,24,4,128,1,float16,float16,0,1.3394559860229491
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,24,8,128,1,float16,float16,0,1.4044431686401366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,24,24,128,1,float16,float16,0,0.8874496459960938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,24,1,128,1,float16,float16,0,0.6818687915802002
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,24,2,128,1,float16,float16,0,0.6883920192718506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,24,24,128,1,float16,float16,0,0.47656641006469724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,24,4,128,1,float16,float16,0,0.696446418762207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,24,8,128,1,float16,float16,0,0.7401631832122803
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,24,1,128,1,float16,float16,0,0.3764879941940308
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,24,2,128,1,float16,float16,0,0.37743520736694336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,24,4,128,1,float16,float16,0,0.379203200340271
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,24,24,128,1,float16,float16,0,0.26872639656066893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,24,8,128,1,float16,float16,0,0.4006608009338379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,24,4,128,1,float16,float16,0,0.22451999187469482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,24,1,128,1,float16,float16,0,0.20311999320983887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,24,2,128,1,float16,float16,0,0.215667200088501
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,24,8,128,1,float16,float16,0,0.22949440479278566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,24,24,128,1,float16,float16,0,0.16742559671401977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,24,1,128,1,float16,float16,0,0.14002879858016967
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,24,2,128,1,float16,float16,0,0.1409440040588379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,24,4,128,1,float16,float16,0,0.13575359582901
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,24,8,128,1,float16,float16,0,0.13733760118484498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,24,2,128,1,float16,float16,0,2.3734880447387696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,24,1,128,1,float16,float16,0,2.3559200286865236
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,24,24,128,1,float16,float16,0,3.4691375732421874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,24,1,128,1,float16,float16,0,4.807244873046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,24,2,128,1,float16,float16,0,4.9217887878417965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,24,4,128,1,float16,float16,0,5.133200073242188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,24,4,128,1,float16,float16,0,2.4863759994506838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,24,8,128,1,float16,float16,0,5.447719955444336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,24,8,128,1,float16,float16,0,2.7673295974731444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,24,24,128,1,float16,float16,0,1.7688800811767578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,24,1,128,1,float16,float16,0,1.221615982055664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,24,1,128,1,float16,float16,0,0.6327439785003662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,24,2,128,1,float16,float16,0,0.6380559921264648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,24,24,128,1,float16,float16,0,0.9075728416442871
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,24,2,128,1,float16,float16,0,1.2472399711608886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,24,4,128,1,float16,float16,0,1.28930721282959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,24,8,128,1,float16,float16,0,1.3646368026733398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,24,1,128,1,float16,float16,0,0.333735990524292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,24,2,128,1,float16,float16,0,0.34538400173187256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,24,4,128,1,float16,float16,0,0.6676784038543702
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,24,24,128,1,float16,float16,0,0.26671199798583983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,24,8,128,1,float16,float16,0,0.3845664024353027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,24,4,128,1,float16,float16,0,0.361407995223999
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,24,1,128,1,float16,float16,0,0.1899216055870056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,24,24,128,1,float16,float16,0,0.4623055934906006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,24,8,128,1,float16,float16,0,0.6997087955474853
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,24,2,128,1,float16,float16,0,0.19606080055236816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,24,2,128,1,float16,float16,0,0.11705759763717652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,24,4,128,1,float16,float16,0,0.20378239154815675
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,24,24,128,1,float16,float16,0,0.15484000444412233
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,24,8,128,1,float16,float16,0,0.2133552074432373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,24,4,128,1,float16,float16,0,0.11839359998703003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,24,1,128,1,float16,float16,0,0.11957600116729736
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,24,24,128,1,float16,float16,0,0.0861631989479065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,24,8,128,1,float16,float16,0,0.13319360017776488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,24,8,128,1,float16,float16,0,0.07013599872589112
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,24,4,128,1,float16,float16,0,0.07577440142631531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,24,1,128,1,float16,float16,0,0.06669920086860656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,24,2,128,1,float16,float16,0,0.07845119833946228
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,24,1,128,1,float16,float16,0,1.387497615814209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,24,2,128,1,float16,float16,0,1.445302391052246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,24,4,128,1,float16,float16,0,1.5124768257141112
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,24,24,128,1,float16,float16,0,2.4965744018554688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,24,1,128,1,float16,float16,0,2.834491157531738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,24,2,128,1,float16,float16,0,2.913404846191406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,24,1,128,1,float16,float16,0,0.7169151782989502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,24,4,128,1,float16,float16,0,3.0665056228637697
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,24,24,128,1,float16,float16,0,1.2770848274230957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,24,8,128,1,float16,float16,0,1.7315071105957032
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,24,2,128,1,float16,float16,0,0.7344992160797119
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,24,4,128,1,float16,float16,0,0.8216239929199218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,24,8,128,1,float16,float16,0,3.5319808959960937
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,24,24,128,1,float16,float16,0,0.6597760200500489
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,24,2,128,1,float16,float16,0,0.38474559783935547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,24,1,128,1,float16,float16,0,0.39328000545501707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,24,4,128,1,float16,float16,0,0.424886417388916
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,24,8,128,1,float16,float16,0,0.9263824462890625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,24,8,128,1,float16,float16,0,0.4562191963195801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,24,4,128,1,float16,float16,0,0.2168272018432617
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,24,24,128,1,float16,float16,0,0.19352320432662964
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,24,1,128,1,float16,float16,0,0.12003840208053589
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,24,2,128,1,float16,float16,0,0.20876638889312743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,24,1,128,1,float16,float16,0,0.20999519824981688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,24,8,128,1,float16,float16,0,0.24765920639038086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,24,24,128,1,float16,float16,0,0.347710394859314
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,24,2,128,1,float16,float16,0,0.11984479427337646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,24,4,128,1,float16,float16,0,0.12548960447311402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,24,8,128,1,float16,float16,0,0.13912960290908813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,24,1,128,1,float16,float16,0,0.06840159893035888
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,24,4,128,1,float16,float16,0,0.0787392020225525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,24,24,128,1,float16,float16,0,0.06610080003738403
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,24,24,128,1,float16,float16,0,0.11000800132751465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,24,1,128,1,float16,float16,0,0.04661599993705749
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,24,2,128,1,float16,float16,0,0.04562720060348511
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,24,2,128,1,float16,float16,0,0.08075680136680603
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,24,4,128,1,float16,float16,0,0.04219520092010498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,24,8,128,1,float16,float16,0,0.05045120120048523
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,24,8,128,1,float16,float16,0,0.0852288007736206
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,24,24,128,1,float16,float16,0,0.03752799928188324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,24,1,128,1,float16,float16,0,0.03333759903907776
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,24,2,128,1,float16,float16,0,0.033399999141693115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,24,8,128,1,float16,float16,0,0.036111998558044436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,24,4,128,1,float16,float16,0,0.03396959900856018
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,24,1,128,1,float16,float16,0,0.5022607803344726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,24,2,128,1,float16,float16,0,0.5063663959503174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,24,1,128,1,float16,float16,0,0.9406191825866699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,24,2,128,1,float16,float16,0,0.9875103950500488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,24,24,128,1,float16,float16,0,1.0865519523620606
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,24,4,128,1,float16,float16,0,1.0690863609313965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,24,4,128,1,float16,float16,0,0.5535855770111084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,24,8,128,1,float16,float16,0,1.2934063911437987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,24,1,128,1,float16,float16,0,0.25366721153259275
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,24,24,128,1,float16,float16,0,0.566537618637085
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,24,8,128,1,float16,float16,0,0.663702392578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,24,2,128,1,float16,float16,0,0.2654752016067505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,24,4,128,1,float16,float16,0,0.28655519485473635
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,24,1,128,1,float16,float16,0,0.13975199460983276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,24,2,128,1,float16,float16,0,0.14493600130081177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,24,24,128,1,float16,float16,0,0.30403521060943606
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,24,8,128,1,float16,float16,0,0.34659039974212646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,24,4,128,1,float16,float16,0,0.1565775990486145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,24,1,128,1,float16,float16,0,0.08257279992103576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,24,24,128,1,float16,float16,0,0.16397119760513307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,24,8,128,1,float16,float16,0,0.18679039478302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,24,2,128,1,float16,float16,0,0.08404960036277771
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,24,4,128,1,float16,float16,0,0.09021919965744019
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,24,8,128,1,float16,float16,0,0.10405440330505371
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,24,1,128,1,float16,float16,0,0.05067999958992005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,24,4,128,1,float16,float16,0,0.05627520084381103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,24,24,128,1,float16,float16,0,0.09284639954566956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,24,8,128,1,float16,float16,0,0.0623088002204895
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,24,2,128,1,float16,float16,0,0.0523472011089325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,24,2,128,1,float16,float16,0,0.028347200155258177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,24,1,128,1,float16,float16,0,0.0306768000125885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,24,24,128,1,float16,float16,0,0.051267200708389284
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,24,24,128,1,float16,float16,0,0.02732959985733032
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,24,4,128,1,float16,float16,0,0.03226400017738342
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,24,4,128,1,float16,float16,0,0.024006399512290954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,24,2,128,1,float16,float16,0,0.02336000055074692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,24,1,128,1,float16,float16,0,0.023051199316978455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,24,8,128,1,float16,float16,0,0.03407520055770874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,24,8,128,1,float16,float16,0,0.024985599517822265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,24,24,128,1,float16,float16,0,0.025142401456832886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,24,1,128,1,float16,float16,0,0.022300800681114195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,24,2,128,1,float16,float16,0,0.023161600530147552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,24,4,128,1,float16,float16,0,0.02264160066843033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,24,8,128,1,float16,float16,0,0.023793600499629974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,24,1,128,1,float16,float16,0,0.19784640073776244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,24,2,128,1,float16,float16,0,0.21332480907440185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,24,1,128,1,float16,float16,0,0.37821760177612307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,24,2,128,1,float16,float16,0,0.4071199893951416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,24,4,128,1,float16,float16,0,0.4509615898132324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,24,8,128,1,float16,float16,0,0.5742847919464111
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,24,24,128,1,float16,float16,0,0.5406208038330078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,24,1,128,1,float16,float16,0,0.11245759725570678
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,24,4,128,1,float16,float16,0,0.2392672061920166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,24,2,128,1,float16,float16,0,0.11936000585556031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,24,8,128,1,float16,float16,0,0.3009488105773926
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,24,4,128,1,float16,float16,0,0.13342080116271973
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,24,24,128,1,float16,float16,0,0.2866640090942383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,24,8,128,1,float16,float16,0,0.16764160394668579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,24,1,128,1,float16,float16,0,0.0671280026435852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,24,2,128,1,float16,float16,0,0.06866080164909363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,24,24,128,1,float16,float16,0,0.1556015968322754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,24,4,128,1,float16,float16,0,0.07581440210342408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,24,8,128,1,float16,float16,0,0.0950160026550293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,24,2,128,1,float16,float16,0,0.03983519971370697
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,24,8,128,1,float16,float16,0,0.052457600831985474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,24,4,128,1,float16,float16,0,0.04424160122871399
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,24,1,128,1,float16,float16,0,0.03811360001564026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,24,24,128,1,float16,float16,0,0.08249120116233825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,24,2,128,1,float16,float16,0,0.023647999763488768
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,24,24,128,1,float16,float16,0,0.04160160124301911
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,24,1,128,1,float16,float16,0,0.02194399982690811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,24,4,128,1,float16,float16,0,0.024167999625205994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,24,8,128,1,float16,float16,0,0.025761601328849793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,24,24,128,1,float16,float16,0,0.02251359969377518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,24,2,128,1,float16,float16,0,0.01865279972553253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,24,8,128,1,float16,float16,0,0.019787199795246124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,24,1,128,1,float16,float16,0,0.0186831995844841
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,24,4,128,1,float16,float16,0,0.0191103994846344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,24,1,128,1,float16,float16,0,0.017759999632835387
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,24,2,128,1,float16,float16,0,0.01790879964828491
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,24,24,128,1,float16,float16,0,0.020369599759578704
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,24,4,128,1,float16,float16,0,0.0179407998919487
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,24,24,128,1,float16,float16,0,0.019468800723552705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,24,8,128,1,float16,float16,0,0.018887999653816222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,24,1,128,1,float16,float16,0,0.017705599963665008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,24,2,128,1,float16,float16,0,0.01770720034837723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,24,4,128,1,float16,float16,0,0.01772959977388382
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,24,8,128,1,float16,float16,0,0.018668800592422485
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,24,1,128,1,float16,float16,0,0.18907840251922609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,24,2,128,1,float16,float16,0,0.11381759643554687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,24,1,128,1,float16,float16,0,0.10635679960250854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,24,2,128,1,float16,float16,0,0.20517919063568116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,24,4,128,1,float16,float16,0,0.23365120887756347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,24,24,128,1,float16,float16,0,0.2833375930786133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,24,4,128,1,float16,float16,0,0.13016639947891234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,24,8,128,1,float16,float16,0,0.3022864103317261
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,24,1,128,1,float16,float16,0,0.06380320191383362
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,24,8,128,1,float16,float16,0,0.16371840238571167
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,24,2,128,1,float16,float16,0,0.06648319959640503
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,24,24,128,1,float16,float16,0,0.15335999727249144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,24,8,128,1,float16,float16,0,0.09265599846839905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,24,4,128,1,float16,float16,0,0.07552800178527833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,24,24,128,1,float16,float16,0,0.08436639904975891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,24,2,128,1,float16,float16,0,0.03606719970703125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,24,1,128,1,float16,float16,0,0.03313120007514954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,24,4,128,1,float16,float16,0,0.04062879979610443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,24,1,128,1,float16,float16,0,0.020367999374866486
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,24,8,128,1,float16,float16,0,0.05304319858551025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,24,24,128,1,float16,float16,0,0.04384160041809082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,24,2,128,1,float16,float16,0,0.02038400024175644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,24,4,128,1,float16,float16,0,0.020611199736595153
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,24,8,128,1,float16,float16,0,0.026915198564529418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,24,1,128,1,float16,float16,0,0.01640319973230362
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,24,2,128,1,float16,float16,0,0.01655679941177368
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,24,4,128,1,float16,float16,0,0.016867199540138246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,24,24,128,1,float16,float16,0,0.02122880071401596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,24,8,128,1,float16,float16,0,0.01910399943590164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,24,24,128,1,float16,float16,0,0.01716320067644119
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,24,1,128,1,float16,float16,0,0.015225599706172942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,24,2,128,1,float16,float16,0,0.015412800014019012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,24,4,128,1,float16,float16,0,0.01568160057067871
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,24,24,128,1,float16,float16,0,0.01595039963722229
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,24,8,128,1,float16,float16,0,0.01592479944229126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,24,1,128,1,float16,float16,0,0.015243199467658997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,24,2,128,1,float16,float16,0,0.015188799798488617
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,24,4,128,1,float16,float16,0,0.015167999267578124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,24,8,128,1,float16,float16,0,0.01555359959602356
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,24,24,128,1,float16,float16,0,0.015476800501346588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,24,2,128,1,float16,float16,0,0.014891199767589569
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,24,4,128,1,float16,float16,0,0.014849600195884705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,24,1,128,1,float16,float16,0,0.015097600221633912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,24,8,128,1,float16,float16,0,0.01496800035238266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,24,1,128,1,float16,float16,0,0.10430079698562622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,24,4,128,1,float16,float16,0,0.1392192006111145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,24,1,128,1,float16,float16,0,0.062745600938797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,24,2,128,1,float16,float16,0,0.11175999641418458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,24,8,128,1,float16,float16,0,0.15849440097808837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,24,24,128,1,float16,float16,0,0.18980319499969484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,24,2,128,1,float16,float16,0,0.06635680198669433
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,24,4,128,1,float16,float16,0,0.0791104018688202
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,24,1,128,1,float16,float16,0,0.033323198556900024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,24,8,128,1,float16,float16,0,0.08990880250930786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,24,2,128,1,float16,float16,0,0.03597599864006042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,24,8,128,1,float16,float16,0,0.05268160104751587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,24,4,128,1,float16,float16,0,0.047758400440216064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,24,1,128,1,float16,float16,0,0.020444799959659577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,24,24,128,1,float16,float16,0,0.057848000526428224
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,24,24,128,1,float16,float16,0,0.10136640071868896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,24,4,128,1,float16,float16,0,0.026467201113700867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,24,8,128,1,float16,float16,0,0.027224001288414002
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,24,2,128,1,float16,float16,0,0.020428800582885744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,24,24,128,1,float16,float16,0,0.02892639935016632
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,24,2,128,1,float16,float16,0,0.016476799547672272
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,24,1,128,1,float16,float16,0,0.016297599673271178
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,24,24,128,1,float16,float16,0,0.018987199664115904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,24,1,128,1,float16,float16,0,0.01520639955997467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,24,2,128,1,float16,float16,0,0.015278400480747223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,24,8,128,1,float16,float16,0,0.018464000523090364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,24,4,128,1,float16,float16,0,0.015747199952602386
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,24,24,128,1,float16,float16,0,0.01605599969625473
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,24,8,128,1,float16,float16,0,0.015625600516796113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,24,4,128,1,float16,float16,0,0.01871040016412735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,24,1,128,1,float16,float16,0,0.014871999621391296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,24,2,128,1,float16,float16,0,0.015230399370193482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,24,4,128,1,float16,float16,0,0.0151296004652977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,24,8,128,1,float16,float16,0,0.015297600626945495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,24,24,128,1,float16,float16,0,0.015345600247383118
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,24,1,128,1,float16,float16,0,0.014876799285411834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,24,4,128,1,float16,float16,0,0.015014399588108063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,24,8,128,1,float16,float16,0,0.014822399616241455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,24,2,128,1,float16,float16,0,0.015027199685573579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,24,1,128,1,float16,float16,0,0.014641599357128143
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,24,2,128,1,float16,float16,0,0.014900800585746766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,24,24,128,1,float16,float16,0,0.015052799880504609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,24,4,128,1,float16,float16,0,0.014819200336933135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,24,8,128,1,float16,float16,0,0.014620800316333771
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,24,1,128,1,float16,float16,0,0.06613600254058838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,24,4,128,1,float16,float16,0,0.07888000011444092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,24,1,128,1,float16,float16,0,0.03426559865474701
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,24,2,128,1,float16,float16,0,0.04395999908447266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,24,2,128,1,float16,float16,0,0.07424479722976685
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,24,8,128,1,float16,float16,0,0.11881760358810425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,24,4,128,1,float16,float16,0,0.048712000250816345
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,24,8,128,1,float16,float16,0,0.06723840236663818
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,24,24,128,1,float16,float16,0,0.15514880418777466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,24,1,128,1,float16,float16,0,0.021038399636745454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,24,24,128,1,float16,float16,0,0.0830847978591919
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,24,4,128,1,float16,float16,0,0.026843199133872987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,24,2,128,1,float16,float16,0,0.026771199703216553
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,24,24,128,1,float16,float16,0,0.0434143990278244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,24,8,128,1,float16,float16,0,0.03487040102481842
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,24,1,128,1,float16,float16,0,0.016748799383640288
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,24,2,128,1,float16,float16,0,0.018662400543689728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,24,4,128,1,float16,float16,0,0.018620799481868743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,24,8,128,1,float16,float16,0,0.024564799666404725
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,24,1,128,1,float16,float16,0,0.01523360013961792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,24,2,128,1,float16,float16,0,0.015323199331760406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,24,4,128,1,float16,float16,0,0.015399999916553497
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,24,24,128,1,float16,float16,0,0.02667039930820465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,24,24,128,1,float16,float16,0,0.01786080002784729
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,24,8,128,1,float16,float16,0,0.017254400253295898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,24,1,128,1,float16,float16,0,0.015056000649929046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,24,2,128,1,float16,float16,0,0.015059199929237366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,24,4,128,1,float16,float16,0,0.015048000216484069
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,24,24,128,1,float16,float16,0,0.015516799688339234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,24,2,128,1,float16,float16,0,0.014907200634479523
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,24,1,128,1,float16,float16,0,0.014758400619029999
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,24,4,128,1,float16,float16,0,0.014777599275112152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,24,8,128,1,float16,float16,0,0.015118399262428283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,24,8,128,1,float16,float16,0,0.014580799639225006
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,24,24,128,1,float16,float16,0,0.015083199739456177
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,24,1,128,1,float16,float16,0,0.014777599275112152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,24,2,128,1,float16,float16,0,0.014640000462532044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,24,4,128,1,float16,float16,0,0.014655999839305878
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,24,24,128,1,float16,float16,0,0.015060800313949584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,24,1,128,1,float16,float16,0,0.014732800424098969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,24,4,128,1,float16,float16,0,0.014505599439144135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,24,2,128,1,float16,float16,0,0.014860799908638
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,24,8,128,1,float16,float16,0,0.014481599628925323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,24,8,128,1,float16,float16,0,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,24,1,128,1,float16,float16,0,0.026443201303482055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,24,2,128,1,float16,float16,0,0.03389599919319153
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,24,4,128,1,float16,float16,0,0.05172479748725891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,24,1,128,1,float16,float16,0,0.01868479996919632
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,24,8,128,1,float16,float16,0,0.08890079855918884
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,24,2,128,1,float16,float16,0,0.024600000679492952
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,24,24,128,1,float16,float16,0,0.12107839584350585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,24,4,128,1,float16,float16,0,0.0321263998746872
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,24,8,128,1,float16,float16,0,0.04967199862003326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,24,2,128,1,float16,float16,0,0.017497600615024568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,24,1,128,1,float16,float16,0,0.015697599947452547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,24,8,128,1,float16,float16,0,0.030947199463844298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,24,24,128,1,float16,float16,0,0.06713439822196961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,24,4,128,1,float16,float16,0,0.023384000360965728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,24,24,128,1,float16,float16,0,0.039900800585746764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,24,1,128,1,float16,float16,0,0.014905600249767304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,24,2,128,1,float16,float16,0,0.015182399749755859
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,24,4,128,1,float16,float16,0,0.016862399876117706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,24,24,128,1,float16,float16,0,0.024932800233364104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,24,1,128,1,float16,float16,0,0.014227199554443359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,24,2,128,1,float16,float16,0,0.014209599792957306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,24,8,128,1,float16,float16,0,0.022873599827289582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,24,4,128,1,float16,float16,0,0.014379200339317322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,24,8,128,1,float16,float16,0,0.016187199950218202
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,24,24,128,1,float16,float16,0,0.017108799517154695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,24,1,128,1,float16,float16,0,0.014060799777507783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,24,2,128,1,float16,float16,0,0.014027200639247894
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,24,8,128,1,float16,float16,0,0.014315199851989747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,24,24,128,1,float16,float16,0,0.015161600708961488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,24,4,128,1,float16,float16,0,0.014256000518798828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,24,1,128,1,float16,float16,0,0.01422560065984726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,24,2,128,1,float16,float16,0,0.013846400380134582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,24,4,128,1,float16,float16,0,0.014280000329017639
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,24,8,128,1,float16,float16,0,0.01432960033416748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,24,24,128,1,float16,float16,0,0.014766399562358857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,24,2,128,1,float16,float16,0,0.014113600552082061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,24,4,128,1,float16,float16,0,0.014004799723625182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,24,1,128,1,float16,float16,0,0.014008000493049622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,24,8,128,1,float16,float16,0,0.014051200449466705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,24,24,128,1,float16,float16,0,0.014526399970054626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,24,1,128,1,float16,float16,0,0.013991999626159667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,24,2,128,1,float16,float16,0,0.014020800590515137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,24,4,128,1,float16,float16,0,0.013724799454212188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,24,8,128,1,float16,float16,0,0.013942399621009826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,16,1,128,1,float16,float16,0,20.880778503417968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,16,2,128,1,float16,float16,0,21.541929626464842
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,16,4,128,1,float16,float16,0,21.514788818359374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,16,1,128,1,float16,float16,0,10.778075408935546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,16,8,128,1,float16,float16,0,21.721707153320313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,16,1,128,1,float16,float16,0,42.05387878417969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,16,2,128,1,float16,float16,0,42.669638061523436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,16,4,128,1,float16,float16,0,43.047061157226565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,16,8,128,1,float16,float16,0,42.96310729980469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,16,1,128,1,float16,float16,0,5.498830413818359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,16,2,128,1,float16,float16,0,5.477377700805664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,16,2,128,1,float16,float16,0,11.011398315429688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,16,4,128,1,float16,float16,0,10.744276428222657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,16,8,128,1,float16,float16,0,10.95438232421875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,16,4,128,1,float16,float16,0,5.625281524658203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,16,8,128,1,float16,float16,0,5.708260726928711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,16,1,128,1,float16,float16,0,24.208602905273438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,16,2,128,1,float16,float16,0,23.843814086914062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,16,4,128,1,float16,float16,0,24.148593139648437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,16,8,128,1,float16,float16,0,24.826795959472655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,16,1,128,1,float16,float16,0,12.300259399414063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,16,2,128,1,float16,float16,0,12.030089569091796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,16,4,128,1,float16,float16,0,12.237921905517577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,16,8,128,1,float16,float16,0,12.585463714599609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,16,1,128,1,float16,float16,0,6.03716812133789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,16,2,128,1,float16,float16,0,6.136254501342774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,16,4,128,1,float16,float16,0,6.319655990600586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,16,1,128,1,float16,float16,0,3.1607072830200194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,16,8,128,1,float16,float16,0,6.275196838378906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,16,2,128,1,float16,float16,0,3.221265411376953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,16,4,128,1,float16,float16,0,3.2494991302490233
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,16,8,128,1,float16,float16,0,3.365875244140625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,16,1,128,1,float16,float16,0,16.624903869628906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,16,2,128,1,float16,float16,0,16.937026977539062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,16,4,128,1,float16,float16,0,16.888858032226562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,16,8,128,1,float16,float16,0,17.368939208984376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,16,1,128,1,float16,float16,0,8.500984191894531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,16,2,128,1,float16,float16,0,8.56456298828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,16,4,128,1,float16,float16,0,8.718704223632812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,16,8,128,1,float16,float16,0,8.76088638305664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,16,16,128,1,float16,float16,0,6.502718353271485
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,16,1,128,1,float16,float16,0,4.179364776611328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,16,16,128,1,float16,float16,0,5.8176624298095705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,16,2,128,1,float16,float16,0,4.338977432250976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,16,16,128,1,float16,float16,0,22.227566528320313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,16,16,128,1,float16,float16,0,4.564697647094727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,16,4,128,1,float16,float16,0,4.356246566772461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,16,8,128,1,float16,float16,0,4.5186767578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,16,1,128,1,float16,float16,0,2.201036834716797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,16,2,128,1,float16,float16,0,2.269799995422363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,16,16,128,1,float16,float16,0,2.382249641418457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,16,16,128,1,float16,float16,0,3.3211902618408202
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,16,4,128,1,float16,float16,0,2.3623695373535156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,16,8,128,1,float16,float16,0,2.393320083618164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,16,16,128,1,float16,float16,0,11.660623931884766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,16,16,128,1,float16,float16,0,12.65118865966797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,16,16,128,1,float16,float16,0,8.793910217285156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,16,1,128,1,float16,float16,0,21.38065948486328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,16,2,128,1,float16,float16,0,21.46977844238281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,16,4,128,1,float16,float16,0,21.516482543945312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,16,8,128,1,float16,float16,0,22.238864135742187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,16,1,128,1,float16,float16,0,10.839323425292969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,16,16,128,1,float16,float16,0,5.8854225158691404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,16,16,128,1,float16,float16,0,11.473985290527343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,16,2,128,1,float16,float16,0,10.739636993408203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,16,1,128,1,float16,float16,0,5.352403259277343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,16,2,128,1,float16,float16,0,5.517540740966797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,16,4,128,1,float16,float16,0,10.826287841796875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,16,1,128,1,float16,float16,0,2.8172176361083983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,16,8,128,1,float16,float16,0,11.344331359863281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,16,16,128,1,float16,float16,0,3.0890655517578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,16,4,128,1,float16,float16,0,5.613876724243164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,16,2,128,1,float16,float16,0,2.7867008209228517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,16,8,128,1,float16,float16,0,5.717091369628906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,16,4,128,1,float16,float16,0,2.9183359146118164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,16,1,128,1,float16,float16,0,1.4853360176086425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,16,16,128,1,float16,float16,0,1.6304176330566407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,16,2,128,1,float16,float16,0,1.538326358795166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,16,8,128,1,float16,float16,0,2.9241167068481446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,16,4,128,1,float16,float16,0,1.6415056228637694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,16,8,128,1,float16,float16,0,1.5822688102722169
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,16,1,128,1,float16,float16,0,6.077420806884765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,16,2,128,1,float16,float16,0,6.229079818725586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,16,4,128,1,float16,float16,0,6.243529510498047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,16,16,128,1,float16,float16,0,6.804914855957032
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,16,1,128,1,float16,float16,0,3.109310340881348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,16,2,128,1,float16,float16,0,3.194062423706055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,16,16,128,1,float16,float16,0,3.4574928283691406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,16,1,128,1,float16,float16,0,12.235310363769532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,16,2,128,1,float16,float16,0,12.574407958984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,16,4,128,1,float16,float16,0,12.522083282470703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,16,8,128,1,float16,float16,0,12.571240234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,16,16,128,1,float16,float16,0,1.8057104110717774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,16,8,128,1,float16,float16,0,6.336585617065429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,16,4,128,1,float16,float16,0,3.162723159790039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,16,8,128,1,float16,float16,0,3.311348724365234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,16,1,128,1,float16,float16,0,0.8986063957214355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,16,16,128,1,float16,float16,0,1.0045663833618164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,16,1,128,1,float16,float16,0,1.581984043121338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,16,2,128,1,float16,float16,0,1.6638992309570313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,16,4,128,1,float16,float16,0,1.6926383972167969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,16,8,128,1,float16,float16,0,1.7050800323486328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,16,2,128,1,float16,float16,0,0.9254480361938476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,16,4,128,1,float16,float16,0,0.9920895576477051
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,16,8,128,1,float16,float16,0,0.9680303573608399
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,16,1,128,1,float16,float16,0,5.5205425262451175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,16,2,128,1,float16,float16,0,5.676998519897461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,16,16,128,1,float16,float16,0,6.259694290161133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,16,4,128,1,float16,float16,0,5.60204963684082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,16,2,128,1,float16,float16,0,2.7976800918579103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,16,1,128,1,float16,float16,0,2.7775600433349608
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,16,16,128,1,float16,float16,0,3.2198768615722657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,16,1,128,1,float16,float16,0,11.226219177246094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,16,2,128,1,float16,float16,0,11.127783966064452
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,16,4,128,1,float16,float16,0,11.412073516845703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,16,8,128,1,float16,float16,0,5.79322395324707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,16,16,128,1,float16,float16,0,1.6586799621582031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,16,8,128,1,float16,float16,0,11.691521453857423
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,16,1,128,1,float16,float16,0,1.4774352073669434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,16,2,128,1,float16,float16,0,1.4932095527648925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,16,4,128,1,float16,float16,0,2.8408992767333983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,16,8,128,1,float16,float16,0,2.9330896377563476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,16,16,128,1,float16,float16,0,0.8922127723693848
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,16,4,128,1,float16,float16,0,1.5177871704101562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,16,1,128,1,float16,float16,0,0.8038144111633301
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,16,2,128,1,float16,float16,0,0.8207440376281738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,16,8,128,1,float16,float16,0,1.5191568374633788
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,16,16,128,1,float16,float16,0,0.529420804977417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,16,1,128,1,float16,float16,0,0.4578432083129883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,16,2,128,1,float16,float16,0,0.45572800636291505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,16,4,128,1,float16,float16,0,0.4700496196746826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,16,4,128,1,float16,float16,0,0.8072976112365723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,16,8,128,1,float16,float16,0,0.8439167976379395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,16,8,128,1,float16,float16,0,0.4928112030029297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,16,1,128,1,float16,float16,0,3.1170095443725585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,16,2,128,1,float16,float16,0,3.215840148925781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,16,16,128,1,float16,float16,0,3.6992897033691405
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,16,1,128,1,float16,float16,0,6.322604751586914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,16,2,128,1,float16,float16,0,6.357486343383789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,16,4,128,1,float16,float16,0,6.610787200927734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,16,4,128,1,float16,float16,0,3.2296432495117187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,16,8,128,1,float16,float16,0,6.809600067138672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,16,16,128,1,float16,float16,0,1.8660575866699218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,16,1,128,1,float16,float16,0,1.6209423065185546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,16,16,128,1,float16,float16,0,1.0142623901367187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,16,2,128,1,float16,float16,0,1.647545623779297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,16,1,128,1,float16,float16,0,0.8739199638366699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,16,8,128,1,float16,float16,0,3.35108642578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,16,4,128,1,float16,float16,0,1.6801855087280273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,16,8,128,1,float16,float16,0,1.7142047882080078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,16,16,128,1,float16,float16,0,0.5542799949645996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,16,2,128,1,float16,float16,0,0.8776944160461426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,16,2,128,1,float16,float16,0,0.5084799766540528
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,16,1,128,1,float16,float16,0,0.4971439838409424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,16,4,128,1,float16,float16,0,0.8938608169555664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,16,8,128,1,float16,float16,0,0.917240047454834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,16,4,128,1,float16,float16,0,0.5074079990386963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,16,8,128,1,float16,float16,0,0.5119088172912598
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,16,16,128,1,float16,float16,0,0.3353231906890869
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,16,1,128,1,float16,float16,0,0.2914639949798584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,16,4,128,1,float16,float16,0,0.29804320335388185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,16,2,128,1,float16,float16,0,0.3109328031539917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,16,8,128,1,float16,float16,0,0.29813759326934813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,16,1,128,1,float16,float16,0,2.891323280334473
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,16,2,128,1,float16,float16,0,2.9314607620239257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,16,4,128,1,float16,float16,0,2.966923141479492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,16,16,128,1,float16,float16,0,3.6036815643310547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,16,1,128,1,float16,float16,0,1.495089626312256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,16,16,128,1,float16,float16,0,1.8386575698852539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,16,1,128,1,float16,float16,0,5.770947265625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,16,2,128,1,float16,float16,0,1.5139023780822753
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,16,2,128,1,float16,float16,0,5.832379150390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,16,4,128,1,float16,float16,0,6.077923202514649
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,16,8,128,1,float16,float16,0,6.361308670043945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,16,8,128,1,float16,float16,0,3.2504657745361327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,16,1,128,1,float16,float16,0,0.7782832145690918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,16,2,128,1,float16,float16,0,0.8042127609252929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,16,16,128,1,float16,float16,0,0.9759023666381836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,16,4,128,1,float16,float16,0,1.5329680442810059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,16,4,128,1,float16,float16,0,0.8321984291076661
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,16,1,128,1,float16,float16,0,0.4269455909729004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,16,8,128,1,float16,float16,0,1.6218975067138672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,16,16,128,1,float16,float16,0,0.537340784072876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,16,2,128,1,float16,float16,0,0.4392560005187988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,16,8,128,1,float16,float16,0,0.8667023658752442
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,16,4,128,1,float16,float16,0,0.44833598136901853
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,16,8,128,1,float16,float16,0,0.46076478958129885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,16,1,128,1,float16,float16,0,0.2466912031173706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,16,16,128,1,float16,float16,0,0.30870718955993653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,16,2,128,1,float16,float16,0,0.254150390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,16,1,128,1,float16,float16,0,0.1682927966117859
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,16,4,128,1,float16,float16,0,0.27016799449920653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,16,16,128,1,float16,float16,0,0.185479998588562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,16,8,128,1,float16,float16,0,0.271068811416626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,16,2,128,1,float16,float16,0,0.17158559560775757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,16,8,128,1,float16,float16,0,0.1724511981010437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,16,4,128,1,float16,float16,0,0.1673135995864868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,16,1,128,1,float16,float16,0,1.6462032318115234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,16,2,128,1,float16,float16,0,1.6879344940185548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,16,4,128,1,float16,float16,0,1.7535568237304688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,16,16,128,1,float16,float16,0,2.1908895492553713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,16,1,128,1,float16,float16,0,0.8797887802124024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,16,1,128,1,float16,float16,0,3.3869808197021483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,16,16,128,1,float16,float16,0,1.150270366668701
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,16,2,128,1,float16,float16,0,3.405574417114258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,16,2,128,1,float16,float16,0,0.8829376220703125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,16,4,128,1,float16,float16,0,3.550350570678711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,16,8,128,1,float16,float16,0,3.7641185760498046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,16,8,128,1,float16,float16,0,1.8734560012817383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,16,2,128,1,float16,float16,0,0.4727151870727539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,16,1,128,1,float16,float16,0,0.49860801696777346
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,16,16,128,1,float16,float16,0,0.6075759887695312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,16,4,128,1,float16,float16,0,0.9277183532714843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,16,4,128,1,float16,float16,0,0.5231711864471436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,16,8,128,1,float16,float16,0,0.9983519554138184
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,16,16,128,1,float16,float16,0,0.3426959991455078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,16,1,128,1,float16,float16,0,0.266758394241333
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,16,2,128,1,float16,float16,0,0.2789135932922363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,16,16,128,1,float16,float16,0,0.20978078842163086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,16,8,128,1,float16,float16,0,0.5294000148773194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,16,4,128,1,float16,float16,0,0.2804368019104004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,16,8,128,1,float16,float16,0,0.296561598777771
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,16,1,128,1,float16,float16,0,0.1721519947052002
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,16,4,128,1,float16,float16,0,0.1635696053504944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,16,2,128,1,float16,float16,0,0.15909919738769532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,16,8,128,1,float16,float16,0,0.186080002784729
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,16,16,128,1,float16,float16,0,0.13429759740829467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,16,1,128,1,float16,float16,0,0.09584800004959107
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,16,2,128,1,float16,float16,0,0.11035679578781128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,16,8,128,1,float16,float16,0,0.10188800096511841
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,16,4,128,1,float16,float16,0,0.1063264012336731
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,16,1,128,1,float16,float16,0,1.5820032119750977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,16,2,128,1,float16,float16,0,1.6364927291870117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,16,4,128,1,float16,float16,0,1.6873119354248047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,16,16,128,1,float16,float16,0,2.2921472549438477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,16,1,128,1,float16,float16,0,3.168396759033203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,16,2,128,1,float16,float16,0,3.254657745361328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,16,16,128,1,float16,float16,0,1.1916128158569337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,16,4,128,1,float16,float16,0,3.3903118133544923
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,16,1,128,1,float16,float16,0,0.8337039947509766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,16,8,128,1,float16,float16,0,3.7913089752197267
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,16,8,128,1,float16,float16,0,1.8878944396972657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,16,2,128,1,float16,float16,0,0.8568240165710449
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,16,1,128,1,float16,float16,0,0.4373023986816406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,16,16,128,1,float16,float16,0,0.6234159946441651
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,16,4,128,1,float16,float16,0,0.9230624198913574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,16,8,128,1,float16,float16,0,0.9501232147216797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,16,16,128,1,float16,float16,0,0.33029439449310305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,16,2,128,1,float16,float16,0,0.4636064052581787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,16,1,128,1,float16,float16,0,0.2459007978439331
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,16,4,128,1,float16,float16,0,0.46791839599609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,16,8,128,1,float16,float16,0,0.5031824111938477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,16,2,128,1,float16,float16,0,0.2634176015853882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,16,8,128,1,float16,float16,0,0.27249600887298586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,16,4,128,1,float16,float16,0,0.25536479949951174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,16,1,128,1,float16,float16,0,0.13776960372924804
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,16,2,128,1,float16,float16,0,0.14833120107650757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,16,16,128,1,float16,float16,0,0.11409920454025269
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,16,1,128,1,float16,float16,0,0.09842879772186279
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,16,8,128,1,float16,float16,0,0.17023199796676636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,16,4,128,1,float16,float16,0,0.15060800313949585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,16,16,128,1,float16,float16,0,0.1970576047897339
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,16,2,128,1,float16,float16,0,0.09680799841880798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,16,8,128,1,float16,float16,0,0.09879999756813049
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,16,4,128,1,float16,float16,0,0.0952239990234375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,16,16,128,1,float16,float16,0,0.07370399832725524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,16,1,128,1,float16,float16,0,0.06203039884567261
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,16,2,128,1,float16,float16,0,0.06260960102081299
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,16,8,128,1,float16,float16,0,0.07211679816246033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,16,4,128,1,float16,float16,0,0.07191200256347656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,16,1,128,1,float16,float16,0,1.5379183769226075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,16,2,128,1,float16,float16,0,1.5650064468383789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,16,16,128,1,float16,float16,0,2.3791872024536134
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,16,1,128,1,float16,float16,0,3.0154592514038088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,16,2,128,1,float16,float16,0,3.106939125061035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,16,4,128,1,float16,float16,0,3.3010177612304688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,16,4,128,1,float16,float16,0,1.650129508972168
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,16,8,128,1,float16,float16,0,3.7745262145996095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,16,16,128,1,float16,float16,0,1.1996607780456543
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,16,8,128,1,float16,float16,0,1.8851200103759767
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,16,1,128,1,float16,float16,0,0.7722544193267822
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,16,4,128,1,float16,float16,0,0.8343680381774903
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,16,2,128,1,float16,float16,0,0.7875440120697021
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,16,16,128,1,float16,float16,0,0.6064112186431885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,16,1,128,1,float16,float16,0,0.2105151891708374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,16,8,128,1,float16,float16,0,0.9493568420410157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,16,16,128,1,float16,float16,0,0.31399199962615965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,16,2,128,1,float16,float16,0,0.40610079765319823
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,16,4,128,1,float16,float16,0,0.42947998046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,16,1,128,1,float16,float16,0,0.39687678813934324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,16,2,128,1,float16,float16,0,0.2155535936355591
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,16,8,128,1,float16,float16,0,0.48555679321289064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,16,4,128,1,float16,float16,0,0.2256704092025757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,16,16,128,1,float16,float16,0,0.16796319484710692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,16,2,128,1,float16,float16,0,0.11887999773025512
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,16,8,128,1,float16,float16,0,0.25464959144592286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,16,4,128,1,float16,float16,0,0.12435359954833984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,16,1,128,1,float16,float16,0,0.11648000478744507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,16,16,128,1,float16,float16,0,0.09601280093193054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,16,8,128,1,float16,float16,0,0.1377568006515503
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,16,1,128,1,float16,float16,0,0.06843839883804322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,16,2,128,1,float16,float16,0,0.0693120002746582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,16,4,128,1,float16,float16,0,0.07384960055351257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,16,1,128,1,float16,float16,0,0.04260160028934479
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,16,8,128,1,float16,float16,0,0.08032320141792297
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,16,16,128,1,float16,float16,0,0.05920000076293945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,16,2,128,1,float16,float16,0,0.04312480092048645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,16,4,128,1,float16,float16,0,0.043942400813102724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,16,16,128,1,float16,float16,0,0.060057598352432254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,16,8,128,1,float16,float16,0,0.04565120041370392
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,16,2,128,1,float16,float16,0,0.04809919893741608
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,16,1,128,1,float16,float16,0,0.047516798973083495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,16,4,128,1,float16,float16,0,0.0527567982673645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,16,8,128,1,float16,float16,0,0.053527998924255374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,16,1,128,1,float16,float16,0,0.33516321182250974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,16,1,128,1,float16,float16,0,0.6461535930633545
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,16,2,128,1,float16,float16,0,0.357590389251709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,16,4,128,1,float16,float16,0,0.40170559883117674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,16,2,128,1,float16,float16,0,0.686198377609253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,16,16,128,1,float16,float16,0,0.73504638671875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,16,4,128,1,float16,float16,0,0.7676943778991699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,16,8,128,1,float16,float16,0,0.9493136405944824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,16,8,128,1,float16,float16,0,0.4907423973083496
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,16,4,128,1,float16,float16,0,0.21396479606628419
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,16,1,128,1,float16,float16,0,0.18152799606323242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,16,16,128,1,float16,float16,0,0.38534719944000245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,16,2,128,1,float16,float16,0,0.18994560241699218
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,16,16,128,1,float16,float16,0,0.21080319881439208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,16,8,128,1,float16,float16,0,0.26315999031066895
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,16,1,128,1,float16,float16,0,0.10236799716949463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,16,2,128,1,float16,float16,0,0.10773119926452637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,16,4,128,1,float16,float16,0,0.11642400026321412
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,16,16,128,1,float16,float16,0,0.12076159715652465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,16,1,128,1,float16,float16,0,0.06402720212936401
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,16,4,128,1,float16,float16,0,0.07073439955711365
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,16,8,128,1,float16,float16,0,0.1465775966644287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,16,2,128,1,float16,float16,0,0.06613439917564393
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,16,8,128,1,float16,float16,0,0.08256800174713134
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,16,1,128,1,float16,float16,0,0.03887679874897003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,16,16,128,1,float16,float16,0,0.06650239825248719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,16,2,128,1,float16,float16,0,0.03780480027198792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,16,4,128,1,float16,float16,0,0.04243359863758087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,16,16,128,1,float16,float16,0,0.037264001369476316
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,16,8,128,1,float16,float16,0,0.04920800030231476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,16,2,128,1,float16,float16,0,0.028753599524497984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,16,1,128,1,float16,float16,0,0.02956160008907318
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,16,8,128,1,float16,float16,0,0.03215200006961823
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,16,4,128,1,float16,float16,0,0.02826400101184845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,16,16,128,1,float16,float16,0,0.025617599487304688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,16,1,128,1,float16,float16,0,0.02284960001707077
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,16,2,128,1,float16,float16,0,0.02330079972743988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,16,4,128,1,float16,float16,0,0.02332800030708313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,16,8,128,1,float16,float16,0,0.023865599930286408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,16,1,128,1,float16,float16,0,0.022015999257564544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,16,16,128,1,float16,float16,0,0.02441920042037964
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,16,2,128,1,float16,float16,0,0.022510400414466857
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,16,4,128,1,float16,float16,0,0.022841599583625794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,16,8,128,1,float16,float16,0,0.02308479994535446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,16,1,128,1,float16,float16,0,0.26935040950775146
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,16,1,128,1,float16,float16,0,0.14467840194702147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,16,2,128,1,float16,float16,0,0.2949840068817139
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,16,2,128,1,float16,float16,0,0.15972160100936889
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,16,16,128,1,float16,float16,0,0.3666975975036621
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,16,4,128,1,float16,float16,0,0.3439552068710327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,16,8,128,1,float16,float16,0,0.4490208148956299
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,16,4,128,1,float16,float16,0,0.18662400245666505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,16,1,128,1,float16,float16,0,0.08392320275306701
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,16,16,128,1,float16,float16,0,0.19869920015335082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,16,8,128,1,float16,float16,0,0.2376784086227417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,16,2,128,1,float16,float16,0,0.09228479862213135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,16,4,128,1,float16,float16,0,0.10643680095672607
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,16,1,128,1,float16,float16,0,0.05254080295562744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,16,16,128,1,float16,float16,0,0.11122720241546631
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,16,2,128,1,float16,float16,0,0.055497598648071286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,16,8,128,1,float16,float16,0,0.13409119844436646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,16,4,128,1,float16,float16,0,0.061540800333023074
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,16,1,128,1,float16,float16,0,0.030191999673843384
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,16,8,128,1,float16,float16,0,0.07363520264625549
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,16,16,128,1,float16,float16,0,0.0592848002910614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,16,2,128,1,float16,float16,0,0.03014880120754242
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,16,4,128,1,float16,float16,0,0.032764801383018495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,16,16,128,1,float16,float16,0,0.029249599575996398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,16,8,128,1,float16,float16,0,0.042263999581336975
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,16,1,128,1,float16,float16,0,0.02287680059671402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,16,8,128,1,float16,float16,0,0.02433599978685379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,16,2,128,1,float16,float16,0,0.021700799465179443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,16,4,128,1,float16,float16,0,0.023414400219917298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,16,16,128,1,float16,float16,0,0.020654399693012238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,16,1,128,1,float16,float16,0,0.0181551992893219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,16,2,128,1,float16,float16,0,0.018118399381637573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,16,4,128,1,float16,float16,0,0.018638400733470915
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,16,8,128,1,float16,float16,0,0.01879040002822876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,16,16,128,1,float16,float16,0,0.019355200231075287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,16,2,128,1,float16,float16,0,0.017734399437904357
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,16,4,128,1,float16,float16,0,0.017923200130462648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,16,1,128,1,float16,float16,0,0.017720000445842744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,16,16,128,1,float16,float16,0,0.01886560022830963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,16,8,128,1,float16,float16,0,0.018025599420070648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,16,2,128,1,float16,float16,0,0.017604799568653108
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,16,4,128,1,float16,float16,0,0.017497600615024568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,16,8,128,1,float16,float16,0,0.017731200158596038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,16,1,128,1,float16,float16,0,0.017529599368572235
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,16,1,128,1,float16,float16,0,0.13837120532989503
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,16,2,128,1,float16,float16,0,0.1535375952720642
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,16,1,128,1,float16,float16,0,0.08107519745826722
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,16,2,128,1,float16,float16,0,0.08838080167770386
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,16,4,128,1,float16,float16,0,0.1815616011619568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,16,16,128,1,float16,float16,0,0.19496320486068724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,16,8,128,1,float16,float16,0,0.2344896078109741
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,16,4,128,1,float16,float16,0,0.10283039808273316
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,16,1,128,1,float16,float16,0,0.05019199848175049
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,16,8,128,1,float16,float16,0,0.13004640340805054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,16,2,128,1,float16,float16,0,0.05303679704666138
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,16,16,128,1,float16,float16,0,0.10944160223007202
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,16,16,128,1,float16,float16,0,0.05912320017814636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,16,4,128,1,float16,float16,0,0.05856159925460815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,16,1,128,1,float16,float16,0,0.02757279872894287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,16,8,128,1,float16,float16,0,0.07261919975280762
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,16,2,128,1,float16,float16,0,0.028454399108886717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,16,4,128,1,float16,float16,0,0.02957119941711426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,16,2,128,1,float16,float16,0,0.0187376007437706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,16,8,128,1,float16,float16,0,0.040966400504112245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,16,16,128,1,float16,float16,0,0.02969760000705719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,16,1,128,1,float16,float16,0,0.01871519982814789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,16,4,128,1,float16,float16,0,0.01923840045928955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,16,16,128,1,float16,float16,0,0.019303999841213226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,16,8,128,1,float16,float16,0,0.020454399287700653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,16,1,128,1,float16,float16,0,0.015652799606323244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,16,2,128,1,float16,float16,0,0.015697599947452547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,16,8,128,1,float16,float16,0,0.016366399824619293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,16,4,128,1,float16,float16,0,0.015937599539756774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,16,16,128,1,float16,float16,0,0.016091200709342956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,16,2,128,1,float16,float16,0,0.015401600301265717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,16,1,128,1,float16,float16,0,0.01512639969587326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,16,16,128,1,float16,float16,0,0.01539359986782074
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,16,4,128,1,float16,float16,0,0.015452800691127777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,16,2,128,1,float16,float16,0,0.015304000675678253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,16,8,128,1,float16,float16,0,0.015619200468063355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,16,1,128,1,float16,float16,0,0.015028800070285796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,16,4,128,1,float16,float16,0,0.014987200498580933
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,16,8,128,1,float16,float16,0,0.015331199765205384
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,16,16,128,1,float16,float16,0,0.014959999918937683
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,16,1,128,1,float16,float16,0,0.0150736004114151
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,16,2,128,1,float16,float16,0,0.014963200688362122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,16,4,128,1,float16,float16,0,0.014891199767589569
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,16,8,128,1,float16,float16,0,0.014934399724006652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,16,1,128,1,float16,float16,0,0.08020319938659667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,16,2,128,1,float16,float16,0,0.08968960046768189
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,16,4,128,1,float16,float16,0,0.10177119970321655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,16,1,128,1,float16,float16,0,0.05121279954910278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,16,8,128,1,float16,float16,0,0.1471328020095825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,16,16,128,1,float16,float16,0,0.13199199438095094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,16,2,128,1,float16,float16,0,0.053643202781677245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,16,4,128,1,float16,float16,0,0.059166401624679565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,16,8,128,1,float16,float16,0,0.08117759823799134
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,16,16,128,1,float16,float16,0,0.07213760018348694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,16,1,128,1,float16,float16,0,0.028070399165153505
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,16,4,128,1,float16,float16,0,0.029980799555778502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,16,8,128,1,float16,float16,0,0.04734880030155182
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,16,2,128,1,float16,float16,0,0.028721600770950317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,16,16,128,1,float16,float16,0,0.03758879899978638
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,16,1,128,1,float16,float16,0,0.01915999948978424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,16,2,128,1,float16,float16,0,0.01920959949493408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,16,4,128,1,float16,float16,0,0.019724799692630766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,16,8,128,1,float16,float16,0,0.02609120011329651
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,16,16,128,1,float16,float16,0,0.02492000013589859
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,16,1,128,1,float16,float16,0,0.015699200332164764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,16,2,128,1,float16,float16,0,0.01549919992685318
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,16,4,128,1,float16,float16,0,0.01594880074262619
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,16,8,128,1,float16,float16,0,0.018193599581718446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,16,16,128,1,float16,float16,0,0.017998400330543517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,16,1,128,1,float16,float16,0,0.015095999836921692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,16,4,128,1,float16,float16,0,0.015294399857521058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,16,2,128,1,float16,float16,0,0.015321600437164306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,16,16,128,1,float16,float16,0,0.015479999780654907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,16,8,128,1,float16,float16,0,0.015406399965286255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,16,1,128,1,float16,float16,0,0.014929600059986115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,16,2,128,1,float16,float16,0,0.01494240015745163
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,16,4,128,1,float16,float16,0,0.015006400644779205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,16,8,128,1,float16,float16,0,0.014963200688362122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,16,16,128,1,float16,float16,0,0.01494400054216385
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,16,1,128,1,float16,float16,0,0.014830400049686433
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,16,2,128,1,float16,float16,0,0.014854399859905243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,16,4,128,1,float16,float16,0,0.014868800342082978
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,16,8,128,1,float16,float16,0,0.014555199444293976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,16,16,128,1,float16,float16,0,0.014843200147151948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,16,1,128,1,float16,float16,0,0.01481119990348816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,16,2,128,1,float16,float16,0,0.01478399932384491
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,16,4,128,1,float16,float16,0,0.014860799908638
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,16,8,128,1,float16,float16,0,0.014428800344467163
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,16,1,128,1,float16,float16,0,0.05272639989852905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,16,2,128,1,float16,float16,0,0.05520480275154114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,16,4,128,1,float16,float16,0,0.07166720032691956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,16,1,128,1,float16,float16,0,0.029017600417137145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,16,2,128,1,float16,float16,0,0.029497599601745604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,16,8,128,1,float16,float16,0,0.1158735990524292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,16,16,128,1,float16,float16,0,0.10981600284576416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,16,4,128,1,float16,float16,0,0.03741759955883026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,16,8,128,1,float16,float16,0,0.06414399743080139
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,16,16,128,1,float16,float16,0,0.054897600412368776
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,16,1,128,1,float16,float16,0,0.019460800290107726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,16,8,128,1,float16,float16,0,0.03409599959850311
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,16,2,128,1,float16,float16,0,0.019832000136375427
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,16,4,128,1,float16,float16,0,0.025563201308250426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,16,16,128,1,float16,float16,0,0.03298560082912445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,16,1,128,1,float16,float16,0,0.01607840061187744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,16,2,128,1,float16,float16,0,0.01595360040664673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,16,4,128,1,float16,float16,0,0.01835840046405792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,16,8,128,1,float16,float16,0,0.02420479953289032
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,16,16,128,1,float16,float16,0,0.023660799860954283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,16,1,128,1,float16,float16,0,0.015087999403476715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,16,4,128,1,float16,float16,0,0.015105600655078887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,16,2,128,1,float16,float16,0,0.015116800367832185
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,16,16,128,1,float16,float16,0,0.017185600101947786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,16,8,128,1,float16,float16,0,0.017283199727535246
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,16,1,128,1,float16,float16,0,0.01488959938287735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,16,2,128,1,float16,float16,0,0.014903999865055084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,16,4,128,1,float16,float16,0,0.014601600170135499
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,16,8,128,1,float16,float16,0,0.014985600113868713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,16,1,128,1,float16,float16,0,0.015024000406265258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,16,16,128,1,float16,float16,0,0.01488959938287735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,16,2,128,1,float16,float16,0,0.01478240042924881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,16,4,128,1,float16,float16,0,0.014571200311183929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,16,8,128,1,float16,float16,0,0.014603200554847717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,16,16,128,1,float16,float16,0,0.014707200229167938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,16,1,128,1,float16,float16,0,0.014868800342082978
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,16,2,128,1,float16,float16,0,0.014769600331783294
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,16,4,128,1,float16,float16,0,0.014507199823856353
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,16,8,128,1,float16,float16,0,0.014294399321079254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,16,16,128,1,float16,float16,0,0.01446399986743927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,16,1,128,1,float16,float16,0,0.014735999703407287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,16,2,128,1,float16,float16,0,0.014880000054836274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,16,4,128,1,float16,float16,0,0.014404800534248353
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,16,8,128,1,float16,float16,0,0.01435679942369461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,16,1,128,1,float16,float16,0,0.026361599564552307
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,16,2,128,1,float16,float16,0,0.033718401193618776
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,16,4,128,1,float16,float16,0,0.05161280035972595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,16,1,128,1,float16,float16,0,0.018639999628067016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,16,8,128,1,float16,float16,0,0.08880320191383362
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,16,16,128,1,float16,float16,0,0.08650239706039428
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,16,2,128,1,float16,float16,0,0.024391999840736388
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,16,4,128,1,float16,float16,0,0.03195840120315552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,16,2,128,1,float16,float16,0,0.017502400279045104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,16,1,128,1,float16,float16,0,0.015596799552440643
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,16,16,128,1,float16,float16,0,0.04841119945049286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,16,8,128,1,float16,float16,0,0.050020802021026614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,16,4,128,1,float16,float16,0,0.023390400409698486
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,16,16,128,1,float16,float16,0,0.030568000674247742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,16,1,128,1,float16,float16,0,0.01483680009841919
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,16,8,128,1,float16,float16,0,0.030905601382255555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,16,2,128,1,float16,float16,0,0.014945599436759948
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,16,4,128,1,float16,float16,0,0.016891199350357055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,16,8,128,1,float16,float16,0,0.022729599475860597
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,16,1,128,1,float16,float16,0,0.013979199528694152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,16,16,128,1,float16,float16,0,0.02221920043230057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,16,2,128,1,float16,float16,0,0.014276799559593201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,16,4,128,1,float16,float16,0,0.014452800154685974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,16,8,128,1,float16,float16,0,0.01635199934244156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,16,16,128,1,float16,float16,0,0.016438399255275727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,16,2,128,1,float16,float16,0,0.014134399592876434
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,16,1,128,1,float16,float16,0,0.014112000167369843
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,16,4,128,1,float16,float16,0,0.014270399510860444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,16,8,128,1,float16,float16,0,0.014316800236701965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,16,16,128,1,float16,float16,0,0.014721600711345673
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,16,2,128,1,float16,float16,0,0.014071999490261078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,16,1,128,1,float16,float16,0,0.013947199285030364
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,16,8,128,1,float16,float16,0,0.014135999977588654
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,16,4,128,1,float16,float16,0,0.01422239989042282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,16,16,128,1,float16,float16,0,0.014584000408649444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,16,1,128,1,float16,float16,0,0.014126400649547576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,16,2,128,1,float16,float16,0,0.013859200477600097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,16,4,128,1,float16,float16,0,0.014102399349212646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,16,8,128,1,float16,float16,0,0.01406400054693222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,16,16,128,1,float16,float16,0,0.014324800670146942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,16,2,128,1,float16,float16,0,0.013990400731563568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,16,1,128,1,float16,float16,0,0.01393119990825653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,16,4,128,1,float16,float16,0,0.01406240016222
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,16,8,128,1,float16,float16,0,0.01412159949541092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,12,1,128,1,float16,float16,0,15.8880126953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,12,2,128,1,float16,float16,0,15.938832092285157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,12,4,128,1,float16,float16,0,16.058827209472657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,12,1,128,1,float16,float16,0,8.051131439208984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,12,2,128,1,float16,float16,0,8.14119873046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,12,1,128,1,float16,float16,0,4.142678451538086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,12,4,128,1,float16,float16,0,8.267520141601562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,12,2,128,1,float16,float16,0,31.780191040039064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,12,1,128,1,float16,float16,0,32.15342407226562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,12,4,128,1,float16,float16,0,32.24098205566406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,12,2,128,1,float16,float16,0,4.2335968017578125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,12,4,128,1,float16,float16,0,4.3286785125732425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,12,1,128,1,float16,float16,0,9.015150451660157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,12,1,128,1,float16,float16,0,17.993576049804688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,12,4,128,1,float16,float16,0,18.45793762207031
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,12,2,128,1,float16,float16,0,18.479585266113283
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,12,2,128,1,float16,float16,0,9.116458892822266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,12,1,128,1,float16,float16,0,4.689038467407227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,12,2,128,1,float16,float16,0,4.643312072753906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,12,4,128,1,float16,float16,0,9.349497222900391
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,12,4,128,1,float16,float16,0,4.923054504394531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,12,1,128,1,float16,float16,0,2.459448051452637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,12,2,128,1,float16,float16,0,2.5227664947509765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,12,4,128,1,float16,float16,0,2.573398399353027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,12,1,128,1,float16,float16,0,12.895860290527343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,12,2,128,1,float16,float16,0,12.75760498046875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,12,4,128,1,float16,float16,0,13.095574951171875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,12,1,128,1,float16,float16,0,6.467601776123047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,12,2,128,1,float16,float16,0,6.522663879394531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,12,4,128,1,float16,float16,0,6.668325042724609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,12,12,128,1,float16,float16,0,4.809579086303711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,12,1,128,1,float16,float16,0,3.195924758911133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,12,12,128,1,float16,float16,0,16.804681396484376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,12,2,128,1,float16,float16,0,3.254679870605469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,12,12,128,1,float16,float16,0,6.775300598144531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,12,12,128,1,float16,float16,0,1.9231023788452148
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,12,1,128,1,float16,float16,0,1.7453935623168946
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,12,4,128,1,float16,float16,0,3.410451126098633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,12,12,128,1,float16,float16,0,4.405382537841797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,12,2,128,1,float16,float16,0,1.7467504501342774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,12,4,128,1,float16,float16,0,1.8280975341796875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,12,12,128,1,float16,float16,0,3.4958030700683596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,12,12,128,1,float16,float16,0,8.73583984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,12,12,128,1,float16,float16,0,9.5586669921875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,12,1,128,1,float16,float16,0,16.350086975097657
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,12,2,128,1,float16,float16,0,16.060406494140626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,12,4,128,1,float16,float16,0,16.496092224121092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,12,1,128,1,float16,float16,0,8.215164947509766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,12,2,128,1,float16,float16,0,8.232710266113282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,12,4,128,1,float16,float16,0,8.316349029541016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,12,12,128,1,float16,float16,0,2.410723114013672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,12,1,128,1,float16,float16,0,2.13863525390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,12,12,128,1,float16,float16,0,4.513812637329101
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,12,1,128,1,float16,float16,0,4.05504150390625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,12,2,128,1,float16,float16,0,4.195608139038086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,12,12,128,1,float16,float16,0,8.627432250976563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,12,4,128,1,float16,float16,0,4.30756950378418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,12,1,128,1,float16,float16,0,1.1526911735534668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,12,2,128,1,float16,float16,0,1.173855972290039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,12,4,128,1,float16,float16,0,1.2735712051391601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,12,12,128,1,float16,float16,0,1.2842160224914552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,12,2,128,1,float16,float16,0,2.131097602844238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,12,4,128,1,float16,float16,0,2.177670478820801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,12,12,128,1,float16,float16,0,2.5501983642578123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,12,12,128,1,float16,float16,0,5.072107315063477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,12,1,128,1,float16,float16,0,4.623775863647461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,12,2,128,1,float16,float16,0,4.66333122253418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,12,4,128,1,float16,float16,0,4.659185409545898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,12,1,128,1,float16,float16,0,2.3843936920166016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,12,12,128,1,float16,float16,0,2.623534393310547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,12,12,128,1,float16,float16,0,1.3997712135314941
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,12,2,128,1,float16,float16,0,2.405940818786621
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,12,4,128,1,float16,float16,0,2.3814624786376952
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,12,1,128,1,float16,float16,0,9.200784301757812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,12,12,128,1,float16,float16,0,0.8039376258850097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,12,2,128,1,float16,float16,0,9.321791839599609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,12,1,128,1,float16,float16,0,1.2693183898925782
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,12,2,128,1,float16,float16,0,1.3092927932739258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,12,4,128,1,float16,float16,0,9.579228973388672
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,12,4,128,1,float16,float16,0,1.3163328170776367
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,12,1,128,1,float16,float16,0,0.7140031814575195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,12,2,128,1,float16,float16,0,0.7348400115966797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,12,4,128,1,float16,float16,0,0.756279993057251
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,12,12,128,1,float16,float16,0,2.3567312240600584
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,12,1,128,1,float16,float16,0,4.0968975067138675
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,12,2,128,1,float16,float16,0,4.089884948730469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,12,4,128,1,float16,float16,0,4.303249740600586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,12,12,128,1,float16,float16,0,4.7685905456542965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,12,1,128,1,float16,float16,0,2.124446487426758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,12,12,128,1,float16,float16,0,1.2894559860229493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,12,1,128,1,float16,float16,0,1.144159984588623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,12,2,128,1,float16,float16,0,1.142572784423828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,12,2,128,1,float16,float16,0,2.138167953491211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,12,4,128,1,float16,float16,0,2.2179567337036135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,12,12,128,1,float16,float16,0,0.7167856216430664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,12,1,128,1,float16,float16,0,0.6136975765228272
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,12,2,128,1,float16,float16,0,0.6601967811584473
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,12,4,128,1,float16,float16,0,1.1796287536621093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,12,12,128,1,float16,float16,0,0.4311056137084961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,12,4,128,1,float16,float16,0,0.6606575965881347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,12,1,128,1,float16,float16,0,0.3715872049331665
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,12,1,128,1,float16,float16,0,8.299454498291016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,12,2,128,1,float16,float16,0,8.394337463378907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,12,2,128,1,float16,float16,0,0.4018256187438965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,12,4,128,1,float16,float16,0,0.3900736093521118
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,12,4,128,1,float16,float16,0,8.633480072021484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,12,12,128,1,float16,float16,0,1.418015956878662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,12,2,128,1,float16,float16,0,2.3672719955444337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,12,1,128,1,float16,float16,0,2.3502416610717773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,12,4,128,1,float16,float16,0,2.476627159118652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,12,12,128,1,float16,float16,0,2.7606784820556642
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,12,1,128,1,float16,float16,0,1.2408559799194336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,12,2,128,1,float16,float16,0,1.2465567588806152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,12,1,128,1,float16,float16,0,4.792763137817383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,12,12,128,1,float16,float16,0,0.7751247882843018
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,12,4,128,1,float16,float16,0,1.3069600105285644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,12,1,128,1,float16,float16,0,0.6889408111572266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,12,2,128,1,float16,float16,0,0.6934656143188477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,12,2,128,1,float16,float16,0,4.734215927124024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,12,1,128,1,float16,float16,0,0.3778559923171997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,12,4,128,1,float16,float16,0,4.940020751953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,12,4,128,1,float16,float16,0,0.7054687976837158
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,12,12,128,1,float16,float16,0,0.46159682273864744
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,12,2,128,1,float16,float16,0,0.41562881469726565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,12,12,128,1,float16,float16,0,0.26880640983581544
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,12,4,128,1,float16,float16,0,0.4175072193145752
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,12,1,128,1,float16,float16,0,0.26046559810638426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,12,2,128,1,float16,float16,0,0.25743999481201174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,12,4,128,1,float16,float16,0,0.27292160987854003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,12,12,128,1,float16,float16,0,1.3743776321411132
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,12,1,128,1,float16,float16,0,2.1813087463378906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,12,2,128,1,float16,float16,0,2.1911392211914062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,12,4,128,1,float16,float16,0,2.303727912902832
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,12,12,128,1,float16,float16,0,2.7043888092041017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,12,1,128,1,float16,float16,0,1.1116864204406738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,12,2,128,1,float16,float16,0,1.1654975891113282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,12,12,128,1,float16,float16,0,0.7464208126068115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,12,1,128,1,float16,float16,0,0.6314191818237305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,12,4,128,1,float16,float16,0,1.2058320045471191
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,12,1,128,1,float16,float16,0,4.322163009643555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,12,12,128,1,float16,float16,0,0.4112351894378662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,12,1,128,1,float16,float16,0,0.3527616024017334
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,12,2,128,1,float16,float16,0,0.6177216053009034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,12,2,128,1,float16,float16,0,4.504419326782227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,12,4,128,1,float16,float16,0,0.6464543819427491
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,12,2,128,1,float16,float16,0,0.3480720043182373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,12,1,128,1,float16,float16,0,0.21159040927886963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,12,2,128,1,float16,float16,0,0.2115936040878296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,12,4,128,1,float16,float16,0,4.671527862548828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,12,4,128,1,float16,float16,0,0.36885600090026854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,12,12,128,1,float16,float16,0,0.24079520702362062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,12,12,128,1,float16,float16,0,0.14531680345535278
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,12,1,128,1,float16,float16,0,0.1182752013206482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,12,4,128,1,float16,float16,0,0.22985119819641114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,12,2,128,1,float16,float16,0,0.14179840087890624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,12,4,128,1,float16,float16,0,0.1364575982093811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,12,1,128,1,float16,float16,0,1.2579471588134765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,12,2,128,1,float16,float16,0,1.2921648025512695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,12,4,128,1,float16,float16,0,1.3786656379699707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,12,12,128,1,float16,float16,0,1.670840072631836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,12,1,128,1,float16,float16,0,0.6829343795776367
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,12,1,128,1,float16,float16,0,2.546009635925293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,12,12,128,1,float16,float16,0,0.8619824409484863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,12,2,128,1,float16,float16,0,0.6801248073577881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,12,2,128,1,float16,float16,0,2.5464096069335938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,12,4,128,1,float16,float16,0,2.733491134643555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,12,4,128,1,float16,float16,0,0.7371407985687256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,12,12,128,1,float16,float16,0,0.4674623966217041
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,12,1,128,1,float16,float16,0,0.3783008098602295
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,12,2,128,1,float16,float16,0,0.37217280864715574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,12,12,128,1,float16,float16,0,0.27362239360809326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,12,4,128,1,float16,float16,0,0.39250080585479735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,12,12,128,1,float16,float16,0,0.16771520376205445
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,12,1,128,1,float16,float16,0,0.13880640268325806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,12,2,128,1,float16,float16,0,0.2246608018875122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,12,4,128,1,float16,float16,0,0.1362496018409729
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,12,4,128,1,float16,float16,0,0.22997760772705078
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,12,2,128,1,float16,float16,0,0.1371072053909302
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,12,1,128,1,float16,float16,0,0.2226560115814209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,12,12,128,1,float16,float16,0,0.12358880043029785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,12,4,128,1,float16,float16,0,0.11457760334014892
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,12,1,128,1,float16,float16,0,0.08389440178871155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,12,2,128,1,float16,float16,0,0.09538080096244812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,12,1,128,1,float16,float16,0,1.1990495681762696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,12,2,128,1,float16,float16,0,1.2315872192382813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,12,4,128,1,float16,float16,0,1.3383343696594239
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,12,12,128,1,float16,float16,0,1.726036834716797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,12,1,128,1,float16,float16,0,0.6345664024353027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,12,1,128,1,float16,float16,0,2.380998420715332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,12,2,128,1,float16,float16,0,0.6625167846679687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,12,2,128,1,float16,float16,0,2.4435247421264648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,12,12,128,1,float16,float16,0,0.8802080154418945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,12,4,128,1,float16,float16,0,2.6961343765258787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,12,4,128,1,float16,float16,0,0.7073455810546875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,12,12,128,1,float16,float16,0,0.47014880180358887
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,12,2,128,1,float16,float16,0,0.3522047996520996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,12,1,128,1,float16,float16,0,0.358240008354187
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,12,12,128,1,float16,float16,0,0.2593391895294189
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,12,4,128,1,float16,float16,0,0.3969088077545166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,12,1,128,1,float16,float16,0,0.19856959581375122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,12,4,128,1,float16,float16,0,0.209116792678833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,12,12,128,1,float16,float16,0,0.15640159845352172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,12,2,128,1,float16,float16,0,0.11857919692993164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,12,2,128,1,float16,float16,0,0.20170240402221679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,12,1,128,1,float16,float16,0,0.12215520143508911
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,12,4,128,1,float16,float16,0,0.13552000522613525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,12,12,128,1,float16,float16,0,0.08803200125694274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,12,2,128,1,float16,float16,0,0.07589920163154602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,12,4,128,1,float16,float16,0,0.06949599981307983
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,12,1,128,1,float16,float16,0,0.07784799933433532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,12,12,128,1,float16,float16,0,0.057340800762176514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,12,2,128,1,float16,float16,0,0.05400000214576721
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,12,1,128,1,float16,float16,0,0.053585600852966306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,12,4,128,1,float16,float16,0,0.055137598514556886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,12,12,128,1,float16,float16,0,1.7870384216308595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,12,12,128,1,float16,float16,0,0.8990032196044921
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,12,1,128,1,float16,float16,0,2.3119295120239256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,12,1,128,1,float16,float16,0,1.1589776039123536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,12,2,128,1,float16,float16,0,2.4296335220336913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,12,2,128,1,float16,float16,0,1.2293999671936036
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,12,4,128,1,float16,float16,0,2.650823974609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,12,4,128,1,float16,float16,0,1.3324607849121093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,12,12,128,1,float16,float16,0,0.4605088233947754
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,12,2,128,1,float16,float16,0,0.32547359466552733
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,12,1,128,1,float16,float16,0,0.3107824087142944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,12,2,128,1,float16,float16,0,0.6256432056427002
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,12,4,128,1,float16,float16,0,0.6924880027770997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,12,4,128,1,float16,float16,0,0.3597791910171509
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,12,1,128,1,float16,float16,0,0.596288013458252
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,12,2,128,1,float16,float16,0,0.1739967942237854
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,12,1,128,1,float16,float16,0,0.16734399795532226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,12,12,128,1,float16,float16,0,0.2436880111694336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,12,12,128,1,float16,float16,0,0.13441120386123656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,12,2,128,1,float16,float16,0,0.09986400008201599
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,12,4,128,1,float16,float16,0,0.1907263994216919
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,12,4,128,1,float16,float16,0,0.109169602394104
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,12,12,128,1,float16,float16,0,0.07893440127372742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,12,1,128,1,float16,float16,0,0.09735040068626404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,12,1,128,1,float16,float16,0,0.05793439745903015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,12,4,128,1,float16,float16,0,0.06502079963684082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,12,2,128,1,float16,float16,0,0.06305760145187378
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,12,12,128,1,float16,float16,0,0.04642080068588257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,12,1,128,1,float16,float16,0,0.04159359931945801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,12,4,128,1,float16,float16,0,0.04272159934043884
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,12,12,128,1,float16,float16,0,0.04437279999256134
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,12,2,128,1,float16,float16,0,0.04158239960670471
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,12,1,128,1,float16,float16,0,0.04132480025291443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,12,2,128,1,float16,float16,0,0.04140639901161194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,12,4,128,1,float16,float16,0,0.042105600237846375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,12,1,128,1,float16,float16,0,0.26281280517578126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,12,1,128,1,float16,float16,0,0.5014095783233643
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,12,4,128,1,float16,float16,0,0.3452944040298462
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,12,2,128,1,float16,float16,0,0.284769606590271
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,12,12,128,1,float16,float16,0,0.29614078998565674
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,12,1,128,1,float16,float16,0,0.14391039609909057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,12,2,128,1,float16,float16,0,0.5471327781677247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,12,12,128,1,float16,float16,0,0.5616896152496338
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,12,1,128,1,float16,float16,0,0.08427519798278808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,12,4,128,1,float16,float16,0,0.6603248119354248
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,12,2,128,1,float16,float16,0,0.15569280385971068
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,12,12,128,1,float16,float16,0,0.16345759630203247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,12,4,128,1,float16,float16,0,0.1862511992454529
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,12,2,128,1,float16,float16,0,0.09016799926757812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,12,2,128,1,float16,float16,0,0.05556640028953552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,12,12,128,1,float16,float16,0,0.09015840291976929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,12,4,128,1,float16,float16,0,0.10449919700622559
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,12,1,128,1,float16,float16,0,0.05284159779548645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,12,4,128,1,float16,float16,0,0.03366400003433227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,12,4,128,1,float16,float16,0,0.06226400136947632
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,12,12,128,1,float16,float16,0,0.027521601319313048
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,12,2,128,1,float16,float16,0,0.03218559920787811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,12,1,128,1,float16,float16,0,0.023387199640274046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,12,1,128,1,float16,float16,0,0.028574401140213014
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,12,12,128,1,float16,float16,0,0.04952160120010376
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,12,4,128,1,float16,float16,0,0.024663999676704407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,12,2,128,1,float16,float16,0,0.023603199422359465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,12,12,128,1,float16,float16,0,0.02552799880504608
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,12,2,128,1,float16,float16,0,0.022708800435066224
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,12,1,128,1,float16,float16,0,0.022814400494098663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,12,4,128,1,float16,float16,0,0.023844799399375914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,12,12,128,1,float16,float16,0,0.02457440048456192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,12,1,128,1,float16,float16,0,0.022436800599098205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,12,4,128,1,float16,float16,0,0.023447999358177186
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,12,2,128,1,float16,float16,0,0.022300800681114195
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,12,1,128,1,float16,float16,0,0.21440160274505615
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,12,2,128,1,float16,float16,0,0.24099359512329102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,12,1,128,1,float16,float16,0,0.11726239919662476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,12,2,128,1,float16,float16,0,0.1324399948120117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,12,4,128,1,float16,float16,0,0.1654096007347107
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,12,12,128,1,float16,float16,0,0.285262393951416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,12,4,128,1,float16,float16,0,0.30316801071166993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,12,1,128,1,float16,float16,0,0.07131199836730957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,12,2,128,1,float16,float16,0,0.07623040080070495
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,12,12,128,1,float16,float16,0,0.15614399909973145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,12,4,128,1,float16,float16,0,0.0930512011051178
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,12,12,128,1,float16,float16,0,0.0835536003112793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,12,4,128,1,float16,float16,0,0.05243679881095886
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,12,12,128,1,float16,float16,0,0.04134239852428436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,12,1,128,1,float16,float16,0,0.023574399948120116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,12,1,128,1,float16,float16,0,0.040575999021530154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,12,2,128,1,float16,float16,0,0.02409279942512512
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,12,2,128,1,float16,float16,0,0.0448063999414444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,12,4,128,1,float16,float16,0,0.02585119903087616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,12,12,128,1,float16,float16,0,0.022699199616909027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,12,1,128,1,float16,float16,0,0.018726399540901183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,12,4,128,1,float16,float16,0,0.01993599981069565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,12,2,128,1,float16,float16,0,0.01918720006942749
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,12,12,128,1,float16,float16,0,0.020393599569797517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,12,2,128,1,float16,float16,0,0.018116800487041472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,12,1,128,1,float16,float16,0,0.01807519942522049
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,12,4,128,1,float16,float16,0,0.018769599497318268
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,12,2,128,1,float16,float16,0,0.0176256000995636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,12,1,128,1,float16,float16,0,0.01762399971485138
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,12,12,128,1,float16,float16,0,0.019310399889945984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,12,4,128,1,float16,float16,0,0.017844800651073457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,12,12,128,1,float16,float16,0,0.018676799535751343
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,12,1,128,1,float16,float16,0,0.017360000312328337
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,12,4,128,1,float16,float16,0,0.018452799320220946
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,12,2,128,1,float16,float16,0,0.017375999689102174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,12,1,128,1,float16,float16,0,0.1146880030632019
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,12,2,128,1,float16,float16,0,0.12802720069885254
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,12,1,128,1,float16,float16,0,0.06757919788360596
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,12,2,128,1,float16,float16,0,0.0754800021648407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,12,4,128,1,float16,float16,0,0.1636304020881653
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,12,4,128,1,float16,float16,0,0.09350240230560303
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,12,12,128,1,float16,float16,0,0.15314240455627443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,12,12,128,1,float16,float16,0,0.0851855993270874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,12,2,128,1,float16,float16,0,0.04240800142288208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,12,1,128,1,float16,float16,0,0.03670240044593811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,12,4,128,1,float16,float16,0,0.0546176016330719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,12,1,128,1,float16,float16,0,0.020747199654579163
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,12,12,128,1,float16,float16,0,0.04446400105953217
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,12,4,128,1,float16,float16,0,0.027958399057388304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,12,2,128,1,float16,float16,0,0.02154559940099716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,12,12,128,1,float16,float16,0,0.02125120013952255
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,12,1,128,1,float16,float16,0,0.016638399660587312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,12,4,128,1,float16,float16,0,0.01932799965143204
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,12,2,128,1,float16,float16,0,0.016857600212097167
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,12,1,128,1,float16,float16,0,0.015435199439525604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,12,2,128,1,float16,float16,0,0.015561600029468537
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,12,12,128,1,float16,float16,0,0.017105600237846373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,12,4,128,1,float16,float16,0,0.01605280041694641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,12,1,128,1,float16,float16,0,0.01515199989080429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,12,2,128,1,float16,float16,0,0.015203200280666351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,12,12,128,1,float16,float16,0,0.015862399339675905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,12,12,128,1,float16,float16,0,0.015379199385643005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,12,4,128,1,float16,float16,0,0.015401600301265717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,12,1,128,1,float16,float16,0,0.014894400537014008
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,12,4,128,1,float16,float16,0,0.015174399316310882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,12,2,128,1,float16,float16,0,0.014979200065135955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,12,12,128,1,float16,float16,0,0.015169599652290344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,12,1,128,1,float16,float16,0,0.014820800721645355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,12,4,128,1,float16,float16,0,0.014929600059986115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,12,2,128,1,float16,float16,0,0.014825600385665893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,12,1,128,1,float16,float16,0,0.06870560050010681
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,12,2,128,1,float16,float16,0,0.08055520057678223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,12,1,128,1,float16,float16,0,0.037729600071907045
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,12,2,128,1,float16,float16,0,0.048819199204444885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,12,12,128,1,float16,float16,0,0.10348960161209106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,12,4,128,1,float16,float16,0,0.09190080165863038
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,12,4,128,1,float16,float16,0,0.054369598627090454
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,12,1,128,1,float16,float16,0,0.020852799713611602
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,12,2,128,1,float16,float16,0,0.027209600806236266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,12,12,128,1,float16,float16,0,0.05896480083465576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,12,12,128,1,float16,float16,0,0.029227200150489806
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,12,4,128,1,float16,float16,0,0.028121599555015565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,12,2,128,1,float16,float16,0,0.019169600307941438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,12,4,128,1,float16,float16,0,0.019300800561904908
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,12,1,128,1,float16,float16,0,0.01721920073032379
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,12,12,128,1,float16,float16,0,0.01881439983844757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,12,1,128,1,float16,float16,0,0.015459200739860535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,12,2,128,1,float16,float16,0,0.015777599811553956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,12,4,128,1,float16,float16,0,0.015481600165367126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,12,12,128,1,float16,float16,0,0.015996800363063814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,12,2,128,1,float16,float16,0,0.015166400372982025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,12,1,128,1,float16,float16,0,0.015110400319099427
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,12,4,128,1,float16,float16,0,0.0152879998087883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,12,12,128,1,float16,float16,0,0.015307199954986573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,12,2,128,1,float16,float16,0,0.014871999621391296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,12,1,128,1,float16,float16,0,0.014720000326633453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,12,4,128,1,float16,float16,0,0.01485760062932968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,12,1,128,1,float16,float16,0,0.01496639996767044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,12,12,128,1,float16,float16,0,0.015097600221633912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,12,4,128,1,float16,float16,0,0.014711999893188476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,12,1,128,1,float16,float16,0,0.014731200039386749
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,12,2,128,1,float16,float16,0,0.014800000190734863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,12,2,128,1,float16,float16,0,0.014912000298500061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,12,12,128,1,float16,float16,0,0.014737600088119506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,12,4,128,1,float16,float16,0,0.014593599736690522
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,12,1,128,1,float16,float16,0,0.045347198843955994
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,12,2,128,1,float16,float16,0,0.04937280118465424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,12,1,128,1,float16,float16,0,0.02736000120639801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,12,4,128,1,float16,float16,0,0.06899039745330811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,12,12,128,1,float16,float16,0,0.0859391987323761
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,12,12,128,1,float16,float16,0,0.044619199633598325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,12,4,128,1,float16,float16,0,0.035579198598861696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,12,2,128,1,float16,float16,0,0.02773439884185791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,12,1,128,1,float16,float16,0,0.019380800426006317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,12,4,128,1,float16,float16,0,0.025252801179885865
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,12,2,128,1,float16,float16,0,0.019331200420856474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,12,1,128,1,float16,float16,0,0.01602399945259094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,12,12,128,1,float16,float16,0,0.02704800069332123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,12,4,128,1,float16,float16,0,0.017907199263572694
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,12,12,128,1,float16,float16,0,0.018003199994564057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,12,1,128,1,float16,float16,0,0.01515520066022873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,12,2,128,1,float16,float16,0,0.01599839925765991
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,12,2,128,1,float16,float16,0,0.014852799475193024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,12,12,128,1,float16,float16,0,0.015476800501346588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,12,2,128,1,float16,float16,0,0.014790399372577668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,12,1,128,1,float16,float16,0,0.014974400401115417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,12,4,128,1,float16,float16,0,0.015113599598407745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,12,12,128,1,float16,float16,0,0.01501920074224472
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,12,2,128,1,float16,float16,0,0.014497600495815277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,12,4,128,1,float16,float16,0,0.014684799313545226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,12,1,128,1,float16,float16,0,0.01462240070104599
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,12,4,128,1,float16,float16,0,0.014496000111103058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,12,12,128,1,float16,float16,0,0.014982399344444276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,12,1,128,1,float16,float16,0,0.014633600413799287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,12,2,128,1,float16,float16,0,0.014478400349617004
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,12,4,128,1,float16,float16,0,0.023521600663661955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,12,1,128,1,float16,float16,0,0.014633600413799287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,12,12,128,1,float16,float16,0,0.014752000570297241
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,12,2,128,1,float16,float16,0,0.014723199605941772
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,12,4,128,1,float16,float16,0,0.014425599575042724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,12,1,128,1,float16,float16,0,0.02629440128803253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,12,2,128,1,float16,float16,0,0.03375680148601532
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,12,4,128,1,float16,float16,0,0.05133439898490906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,12,12,128,1,float16,float16,0,0.06901440024375916
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,12,2,128,1,float16,float16,0,0.024372799694538115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,12,4,128,1,float16,float16,0,0.03207840025424957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,12,12,128,1,float16,float16,0,0.04106720089912415
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,12,1,128,1,float16,float16,0,0.01871040016412735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,12,2,128,1,float16,float16,0,0.017441600561141968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,12,1,128,1,float16,float16,0,0.015544000267982482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,12,4,128,1,float16,float16,0,0.02322240024805069
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,12,1,128,1,float16,float16,0,0.014718399941921234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,12,12,128,1,float16,float16,0,0.02560960054397583
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,12,2,128,1,float16,float16,0,0.014921599626541137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,12,1,128,1,float16,float16,0,0.014070400595664978
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,12,12,128,1,float16,float16,0,0.016795200109481812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,12,4,128,1,float16,float16,0,0.016892799735069276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,12,2,128,1,float16,float16,0,0.014239999651908874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,12,4,128,1,float16,float16,0,0.014574399590492249
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,12,12,128,1,float16,float16,0,0.015027199685573579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,12,1,128,1,float16,float16,0,0.014078399538993836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,12,2,128,1,float16,float16,0,0.014068800210952758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,12,4,128,1,float16,float16,0,0.014156800508499146
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,12,1,128,1,float16,float16,0,0.01395840048789978
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,12,2,128,1,float16,float16,0,0.014057600498199463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,12,4,128,1,float16,float16,0,0.014246399700641631
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,12,12,128,1,float16,float16,0,0.014582400023937226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,12,12,128,1,float16,float16,0,0.014860799908638
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,12,1,128,1,float16,float16,0,0.013939200341701508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,12,2,128,1,float16,float16,0,0.013876800239086152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,12,4,128,1,float16,float16,0,0.014131200313568116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,12,12,128,1,float16,float16,0,0.014448000490665436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,12,1,128,1,float16,float16,0,0.014188799262046813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,12,2,128,1,float16,float16,0,0.013889600336551667
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,12,4,128,1,float16,float16,0,0.013956800103187561
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,8,1,128,1,float16,float16,0,10.624678039550782
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,8,2,128,1,float16,float16,0,10.807630157470703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,8,8,128,1,float16,float16,0,11.355731201171874
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,8,4,128,1,float16,float16,0,11.208523559570313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,8,8,128,1,float16,float16,0,5.780311965942383
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,8,1,128,1,float16,float16,0,5.482547378540039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,8,2,128,1,float16,float16,0,5.579902267456054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,8,8,128,1,float16,float16,0,3.2129711151123046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,8,4,128,1,float16,float16,0,5.722812652587891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,8,1,128,1,float16,float16,0,2.870968055725098
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,8,2,128,1,float16,float16,0,2.9667407989501955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,8,4,128,1,float16,float16,0,3.064401626586914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,8,1,128,1,float16,float16,0,20.852980041503905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,8,2,128,1,float16,float16,0,21.388975524902342
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,8,4,128,1,float16,float16,0,21.48169708251953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,8,8,128,1,float16,float16,0,3.3888175964355467
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,8,8,128,1,float16,float16,0,6.463593292236328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,8,1,128,1,float16,float16,0,6.1398368835449215
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,8,2,128,1,float16,float16,0,6.1580047607421875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,8,4,128,1,float16,float16,0,6.375644683837891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,8,1,128,1,float16,float16,0,3.1704864501953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,8,8,128,1,float16,float16,0,1.8822416305541991
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,8,1,128,1,float16,float16,0,12.264030456542969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,8,2,128,1,float16,float16,0,12.285307312011719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,8,2,128,1,float16,float16,0,3.220150375366211
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,8,4,128,1,float16,float16,0,3.318899154663086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,8,1,128,1,float16,float16,0,1.7618080139160157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,8,4,128,1,float16,float16,0,12.406886291503906
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,8,2,128,1,float16,float16,0,1.7544031143188477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,8,4,128,1,float16,float16,0,1.820756721496582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,8,1,128,1,float16,float16,0,4.322987365722656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,8,8,128,1,float16,float16,0,2.350726318359375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,8,8,128,1,float16,float16,0,4.593289566040039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,8,2,128,1,float16,float16,0,4.375436782836914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,8,4,128,1,float16,float16,0,4.394198226928711
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,8,8,128,1,float16,float16,0,1.3709039688110352
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,8,2,128,1,float16,float16,0,2.299737548828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,8,1,128,1,float16,float16,0,2.406064033508301
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,8,4,128,1,float16,float16,0,2.3795679092407225
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,8,1,128,1,float16,float16,0,8.627056121826172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,8,4,128,1,float16,float16,0,8.615264129638671
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,8,2,128,1,float16,float16,0,8.659352111816407
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,8,1,128,1,float16,float16,0,1.2371439933776855
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,8,2,128,1,float16,float16,0,1.236520004272461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,8,4,128,1,float16,float16,0,1.3209952354431151
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,8,8,128,1,float16,float16,0,2.9480623245239257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,8,1,128,1,float16,float16,0,5.482207870483398
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,8,2,128,1,float16,float16,0,5.503193664550781
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,8,8,128,1,float16,float16,0,5.801811218261719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,8,4,128,1,float16,float16,0,5.637768173217774
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,8,1,128,1,float16,float16,0,2.7709728240966798
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,8,8,128,1,float16,float16,0,1.6261215209960938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,8,1,128,1,float16,float16,0,1.535364818572998
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,8,2,128,1,float16,float16,0,1.56702880859375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,8,2,128,1,float16,float16,0,2.9227872848510743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,8,4,128,1,float16,float16,0,2.9110895156860352
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,8,8,128,1,float16,float16,0,0.9383664131164551
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,8,4,128,1,float16,float16,0,1.550203227996826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,8,1,128,1,float16,float16,0,0.8737039566040039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,8,2,128,1,float16,float16,0,0.8872655868530274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,8,4,128,1,float16,float16,0,0.9240351676940918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,8,1,128,1,float16,float16,0,10.831511688232421
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,8,2,128,1,float16,float16,0,10.943417358398438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,8,4,128,1,float16,float16,0,11.31534423828125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,8,1,128,1,float16,float16,0,3.1173648834228516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,8,8,128,1,float16,float16,0,1.8291696548461913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,8,8,128,1,float16,float16,0,3.4164718627929687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,8,2,128,1,float16,float16,0,3.2152576446533203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,8,4,128,1,float16,float16,0,3.2659278869628907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,8,8,128,1,float16,float16,0,1.0111424446105957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,8,1,128,1,float16,float16,0,0.9280495643615723
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,8,1,128,1,float16,float16,0,1.6820079803466796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,8,2,128,1,float16,float16,0,1.6766271591186523
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,8,4,128,1,float16,float16,0,1.7369535446166993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,8,1,128,1,float16,float16,0,6.1895294189453125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,8,8,128,1,float16,float16,0,0.6219520092010498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,8,2,128,1,float16,float16,0,6.222886276245117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,8,1,128,1,float16,float16,0,0.5702191829681397
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,8,4,128,1,float16,float16,0,6.312763214111328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,8,4,128,1,float16,float16,0,0.9529871940612793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,8,2,128,1,float16,float16,0,0.949886417388916
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,8,2,128,1,float16,float16,0,0.5863408088684082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,8,4,128,1,float16,float16,0,0.5887087821960449
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,8,8,128,1,float16,float16,0,1.6851823806762696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,8,1,128,1,float16,float16,0,2.7972864151000976
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,8,2,128,1,float16,float16,0,2.8446367263793944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,8,4,128,1,float16,float16,0,2.9194303512573243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,8,8,128,1,float16,float16,0,3.1112895965576173
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,8,1,128,1,float16,float16,0,1.4543408393859862
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,8,8,128,1,float16,float16,0,0.900051212310791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,8,1,128,1,float16,float16,0,0.8146544456481933
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,8,2,128,1,float16,float16,0,1.5167776107788087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,8,2,128,1,float16,float16,0,0.8116031646728515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,8,4,128,1,float16,float16,0,1.5552576065063477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,8,8,128,1,float16,float16,0,0.51387038230896
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,8,1,128,1,float16,float16,0,0.4608176231384277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,8,4,128,1,float16,float16,0,0.846895980834961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,8,1,128,1,float16,float16,0,5.525046539306641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,8,2,128,1,float16,float16,0,0.4620880126953125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,8,4,128,1,float16,float16,0,0.4870255947113037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,8,8,128,1,float16,float16,0,0.3311408042907715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,8,2,128,1,float16,float16,0,5.604574584960938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,8,1,128,1,float16,float16,0,0.31601920127868655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,8,4,128,1,float16,float16,0,0.3091599941253662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,8,4,128,1,float16,float16,0,5.901561737060547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,8,2,128,1,float16,float16,0,0.32804160118103026
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,8,1,128,1,float16,float16,0,1.593729591369629
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,8,2,128,1,float16,float16,0,1.6673999786376954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,8,8,128,1,float16,float16,0,1.8591295242309571
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,8,4,128,1,float16,float16,0,1.688694381713867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,8,8,128,1,float16,float16,0,0.998408031463623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,8,1,128,1,float16,float16,0,3.1955535888671873
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,8,1,128,1,float16,float16,0,0.889252758026123
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,8,2,128,1,float16,float16,0,3.237636947631836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,8,2,128,1,float16,float16,0,0.8899807929992676
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,8,4,128,1,float16,float16,0,0.9211088180541992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,8,4,128,1,float16,float16,0,3.3309505462646483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,8,8,128,1,float16,float16,0,0.5493824005126953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,8,8,128,1,float16,float16,0,0.34275040626525877
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,8,2,128,1,float16,float16,0,0.4975567817687988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,8,1,128,1,float16,float16,0,0.509822416305542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,8,1,128,1,float16,float16,0,0.3074480056762695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,8,2,128,1,float16,float16,0,0.31636319160461424
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,8,4,128,1,float16,float16,0,0.5306335926055908
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,8,8,128,1,float16,float16,0,0.19177440404891968
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,8,1,128,1,float16,float16,0,0.1699295997619629
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,8,2,128,1,float16,float16,0,0.20980160236358641
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,8,4,128,1,float16,float16,0,0.3023135900497437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,8,4,128,1,float16,float16,0,0.19878560304641724
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,8,1,128,1,float16,float16,0,1.462507152557373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,8,2,128,1,float16,float16,0,1.4926336288452149
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,8,4,128,1,float16,float16,0,1.5695712089538574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,8,8,128,1,float16,float16,0,1.8074623107910157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,8,1,128,1,float16,float16,0,0.7859551906585693
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,8,2,128,1,float16,float16,0,0.8066847801208497
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,8,1,128,1,float16,float16,0,2.9020879745483397
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,8,8,128,1,float16,float16,0,0.9401280403137207
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,8,2,128,1,float16,float16,0,2.997616004943848
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,8,4,128,1,float16,float16,0,0.8621472358703614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,8,4,128,1,float16,float16,0,3.1549327850341795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,8,1,128,1,float16,float16,0,0.43025760650634765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,8,2,128,1,float16,float16,0,0.4396575927734375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,8,8,128,1,float16,float16,0,0.4989615917205811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,8,2,128,1,float16,float16,0,0.2665776014328003
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,8,1,128,1,float16,float16,0,0.2568079948425293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,8,8,128,1,float16,float16,0,0.1767392039299011
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,8,8,128,1,float16,float16,0,0.29437758922576907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,8,4,128,1,float16,float16,0,0.4676784038543701
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,8,4,128,1,float16,float16,0,0.28242719173431396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,8,1,128,1,float16,float16,0,0.16929759979248046
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,8,4,128,1,float16,float16,0,0.17776639461517335
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,8,2,128,1,float16,float16,0,0.1641535997390747
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,8,8,128,1,float16,float16,0,0.13829439878463745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,8,1,128,1,float16,float16,0,0.10203360319137574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,8,2,128,1,float16,float16,0,0.11033600568771362
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,8,4,128,1,float16,float16,0,0.1284719944000244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,8,8,128,1,float16,float16,0,0.5922143936157227
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,8,1,128,1,float16,float16,0,0.8874032020568847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,8,2,128,1,float16,float16,0,0.9000080108642579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,8,4,128,1,float16,float16,0,0.9559359550476074
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,8,8,128,1,float16,float16,0,1.1152000427246094
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,8,1,128,1,float16,float16,0,1.716953659057617
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,8,1,128,1,float16,float16,0,0.468609619140625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,8,2,128,1,float16,float16,0,0.48942880630493163
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,8,8,128,1,float16,float16,0,0.3286319971084595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,8,1,128,1,float16,float16,0,0.26344320774078367
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,8,4,128,1,float16,float16,0,0.510214376449585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,8,2,128,1,float16,float16,0,1.7515199661254883
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,8,4,128,1,float16,float16,0,1.8749744415283203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,8,2,128,1,float16,float16,0,0.16337440013885499
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,8,2,128,1,float16,float16,0,0.2886080026626587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,8,4,128,1,float16,float16,0,0.28855841159820556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,8,8,128,1,float16,float16,0,0.19722559452056884
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,8,1,128,1,float16,float16,0,0.1567792057991028
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,8,8,128,1,float16,float16,0,0.1337551951408386
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,8,4,128,1,float16,float16,0,0.17716959714889527
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,8,1,128,1,float16,float16,0,0.0728048026561737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,8,4,128,1,float16,float16,0,0.09940959811210633
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,8,2,128,1,float16,float16,0,0.1038432002067566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,8,1,128,1,float16,float16,0,0.11071360111236572
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,8,8,128,1,float16,float16,0,0.0769919991493225
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,8,4,128,1,float16,float16,0,0.07427039742469788
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,8,2,128,1,float16,float16,0,0.07229120135307313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,8,8,128,1,float16,float16,0,0.7811344146728516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,8,1,128,1,float16,float16,0,1.1746416091918945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,8,2,128,1,float16,float16,0,1.2233535766601562
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,8,4,128,1,float16,float16,0,1.3299535751342773
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,8,8,128,1,float16,float16,0,1.5437040328979492
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,8,1,128,1,float16,float16,0,2.308198356628418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,8,1,128,1,float16,float16,0,0.5991568088531494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,8,2,128,1,float16,float16,0,0.6356527805328369
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,8,8,128,1,float16,float16,0,0.4099167823791504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,8,1,128,1,float16,float16,0,0.31480000019073484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,8,4,128,1,float16,float16,0,0.6765535831451416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,8,2,128,1,float16,float16,0,2.4180271148681642
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,8,2,128,1,float16,float16,0,0.3318480014801025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,8,8,128,1,float16,float16,0,0.22134239673614503
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,8,4,128,1,float16,float16,0,0.35417120456695556
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,8,4,128,1,float16,float16,0,2.63430233001709
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,8,1,128,1,float16,float16,0,0.17432479858398436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,8,2,128,1,float16,float16,0,0.18099839687347413
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,8,4,128,1,float16,float16,0,0.19370559453964234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,8,8,128,1,float16,float16,0,0.12547199726104735
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,8,2,128,1,float16,float16,0,0.10743680000305175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,8,1,128,1,float16,float16,0,0.10459200143814087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,8,8,128,1,float16,float16,0,0.07613760232925415
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,8,1,128,1,float16,float16,0,0.06472319960594178
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,8,2,128,1,float16,float16,0,0.0653711974620819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,8,4,128,1,float16,float16,0,0.11428320407867432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,8,8,128,1,float16,float16,0,0.06901919841766357
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,8,1,128,1,float16,float16,0,0.05375679731369019
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,8,4,128,1,float16,float16,0,0.06725760102272034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,8,2,128,1,float16,float16,0,0.05351520180702209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,8,4,128,1,float16,float16,0,0.06601600050926208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,8,1,128,1,float16,float16,0,0.7973648071289062
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,8,2,128,1,float16,float16,0,0.8564944267272949
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,8,4,128,1,float16,float16,0,0.9406656265258789
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,8,8,128,1,float16,float16,0,1.2014368057250977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,8,1,128,1,float16,float16,0,1.566808032989502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,8,2,128,1,float16,float16,0,1.6853567123413087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,8,4,128,1,float16,float16,0,1.8673904418945313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,8,1,128,1,float16,float16,0,0.4150224208831787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,8,2,128,1,float16,float16,0,0.44251837730407717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,8,8,128,1,float16,float16,0,0.6135903835296631
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,8,8,128,1,float16,float16,0,0.31917920112609866
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,8,8,128,1,float16,float16,0,0.17216960191726685
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,8,1,128,1,float16,float16,0,0.2217855930328369
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,8,1,128,1,float16,float16,0,0.12343039512634277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,8,2,128,1,float16,float16,0,0.23480479717254638
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,8,4,128,1,float16,float16,0,0.25725440979003905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,8,4,128,1,float16,float16,0,0.4851600170135498
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,8,2,128,1,float16,float16,0,0.13004640340805054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,8,8,128,1,float16,float16,0,0.0997600018978119
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,8,4,128,1,float16,float16,0,0.14205440282821655
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,8,1,128,1,float16,float16,0,0.0728048026561737
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,8,2,128,1,float16,float16,0,0.07748159766197205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,8,1,128,1,float16,float16,0,0.04388799965381622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,8,4,128,1,float16,float16,0,0.0832431972026825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,8,8,128,1,float16,float16,0,0.06384959816932678
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,8,4,128,1,float16,float16,0,0.04595359861850738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,8,2,128,1,float16,float16,0,0.04521600008010864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,8,1,128,1,float16,float16,0,0.04800800085067749
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,8,8,128,1,float16,float16,0,0.06011360287666321
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,8,4,128,1,float16,float16,0,0.05401759743690491
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,8,2,128,1,float16,float16,0,0.05203199982643127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,8,1,128,1,float16,float16,0,0.03518239855766296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,8,2,128,1,float16,float16,0,0.03571679890155792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,8,8,128,1,float16,float16,0,0.0366703987121582
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,8,4,128,1,float16,float16,0,0.03566400110721588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,8,1,128,1,float16,float16,0,0.35714399814605713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,8,8,128,1,float16,float16,0,0.381662392616272
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,8,2,128,1,float16,float16,0,0.21540799140930175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,8,1,128,1,float16,float16,0,0.19245439767837524
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,8,1,128,1,float16,float16,0,0.10831520557403565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,8,2,128,1,float16,float16,0,0.40428638458251953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,8,8,128,1,float16,float16,0,0.20639519691467284
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,8,4,128,1,float16,float16,0,0.25909759998321535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,8,2,128,1,float16,float16,0,0.12005280256271363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,8,1,128,1,float16,float16,0,0.06622560024261474
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,8,8,128,1,float16,float16,0,0.11794719696044922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,8,4,128,1,float16,float16,0,0.4882768154144287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,8,4,128,1,float16,float16,0,0.14370399713516235
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,8,2,128,1,float16,float16,0,0.07186560034751892
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,8,2,128,1,float16,float16,0,0.04232960045337677
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,8,4,128,1,float16,float16,0,0.08174239993095397
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,8,4,128,1,float16,float16,0,0.048814401030540466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,8,8,128,1,float16,float16,0,0.06498240232467652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,8,1,128,1,float16,float16,0,0.02869440019130707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,8,1,128,1,float16,float16,0,0.03815360069274902
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,8,4,128,1,float16,float16,0,0.03238399922847748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,8,2,128,1,float16,float16,0,0.028281599283218384
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,8,8,128,1,float16,float16,0,0.038787201046943665
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,8,8,128,1,float16,float16,0,0.02560960054397583
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,8,1,128,1,float16,float16,0,0.02327840030193329
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,8,2,128,1,float16,float16,0,0.02327519953250885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,8,4,128,1,float16,float16,0,0.023694400489330292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,8,2,128,1,float16,float16,0,0.022865599393844603
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,8,8,128,1,float16,float16,0,0.02438559979200363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,8,1,128,1,float16,float16,0,0.022668799757957457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,8,1,128,1,float16,float16,0,0.02260800004005432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,8,4,128,1,float16,float16,0,0.022945599257946016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,8,8,128,1,float16,float16,0,0.023556800186634065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,8,2,128,1,float16,float16,0,0.022519999742507936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,8,4,128,1,float16,float16,0,0.022652800381183624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,8,1,128,1,float16,float16,0,0.16165440082550048
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,8,1,128,1,float16,float16,0,0.09073439836502076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,8,2,128,1,float16,float16,0,0.10543520450592041
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,8,2,128,1,float16,float16,0,0.18652000427246093
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,8,8,128,1,float16,float16,0,0.11113120317459106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,8,8,128,1,float16,float16,0,0.19724160432815552
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,8,4,128,1,float16,float16,0,0.13029439449310304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,8,1,128,1,float16,float16,0,0.05597280263900757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,8,4,128,1,float16,float16,0,0.2357680082321167
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,8,2,128,1,float16,float16,0,0.061603200435638425
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,8,8,128,1,float16,float16,0,0.06000000238418579
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,8,4,128,1,float16,float16,0,0.07277119755744935
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,8,8,128,1,float16,float16,0,0.029652801156044007
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,8,4,128,1,float16,float16,0,0.04284639954566956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,8,2,128,1,float16,float16,0,0.02338559925556183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,8,1,128,1,float16,float16,0,0.031248000264167786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,8,2,128,1,float16,float16,0,0.03248319923877716
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,8,1,128,1,float16,float16,0,0.021958400309085847
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,8,8,128,1,float16,float16,0,0.020791999995708466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,8,4,128,1,float16,float16,0,0.024327999353408812
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,8,2,128,1,float16,float16,0,0.018374399840831758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,8,1,128,1,float16,float16,0,0.01828960031270981
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,8,4,128,1,float16,float16,0,0.0188960000872612
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,8,8,128,1,float16,float16,0,0.019100800156593323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,8,1,128,1,float16,float16,0,0.018163199722766876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,8,2,128,1,float16,float16,0,0.017899200320243835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,8,4,128,1,float16,float16,0,0.01809599995613098
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,8,8,128,1,float16,float16,0,0.018596799671649934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,8,1,128,1,float16,float16,0,0.017664000391960144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,8,4,128,1,float16,float16,0,0.017795200645923614
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,8,2,128,1,float16,float16,0,0.0176816001534462
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,8,8,128,1,float16,float16,0,0.018343999981880188
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,8,1,128,1,float16,float16,0,0.017401599884033205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,8,2,128,1,float16,float16,0,0.01722719967365265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,8,4,128,1,float16,float16,0,0.017451199889183044
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,8,1,128,1,float16,float16,0,0.09038879871368408
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,8,2,128,1,float16,float16,0,0.10355839729309083
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,8,2,128,1,float16,float16,0,0.059910398721694944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,8,1,128,1,float16,float16,0,0.05466880202293396
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,8,8,128,1,float16,float16,0,0.10791679620742797
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,8,4,128,1,float16,float16,0,0.07296159863471985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,8,4,128,1,float16,float16,0,0.1299023985862732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,8,8,128,1,float16,float16,0,0.060192000865936277
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,8,2,128,1,float16,float16,0,0.03091199994087219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,8,1,128,1,float16,float16,0,0.029260799288749695
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,8,4,128,1,float16,float16,0,0.0416703999042511
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,8,8,128,1,float16,float16,0,0.030740800499916076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,8,1,128,1,float16,float16,0,0.01943040043115616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,8,4,128,1,float16,float16,0,0.02083680033683777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,8,8,128,1,float16,float16,0,0.01926400065422058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,8,2,128,1,float16,float16,0,0.016172799468040466
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,8,1,128,1,float16,float16,0,0.015852800011634825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,8,2,128,1,float16,float16,0,0.02003840059041977
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,8,8,128,1,float16,float16,0,0.015932799875736238
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,8,1,128,1,float16,float16,0,0.015430399775505066
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,8,4,128,1,float16,float16,0,0.016484799981117248
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,8,2,128,1,float16,float16,0,0.015430399775505066
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,8,4,128,1,float16,float16,0,0.015699200332164764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,8,8,128,1,float16,float16,0,0.015107199549674988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,8,4,128,1,float16,float16,0,0.01525920033454895
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,8,2,128,1,float16,float16,0,0.01496800035238266
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,8,1,128,1,float16,float16,0,0.014903999865055084
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,8,8,128,1,float16,float16,0,0.015086400508880615
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,8,2,128,1,float16,float16,0,0.014959999918937683
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,8,1,128,1,float16,float16,0,0.015150399506092071
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,8,4,128,1,float16,float16,0,0.014735999703407287
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,8,8,128,1,float16,float16,0,0.014596800506114959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,8,1,128,1,float16,float16,0,0.014871999621391296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,8,2,128,1,float16,float16,0,0.014788800477981567
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,8,4,128,1,float16,float16,0,0.015097600221633912
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,8,1,128,1,float16,float16,0,0.029651200771331786
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,8,2,128,1,float16,float16,0,0.06091359853744507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,8,1,128,1,float16,float16,0,0.05604000091552734
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,8,2,128,1,float16,float16,0,0.03134559988975525
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,8,4,128,1,float16,float16,0,0.04827359914779663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,8,8,128,1,float16,float16,0,0.07389280200004578
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,8,4,128,1,float16,float16,0,0.08222079873085023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,8,1,128,1,float16,float16,0,0.01987999975681305
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,8,8,128,1,float16,float16,0,0.03907999992370605
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,8,4,128,1,float16,float16,0,0.026643198728561402
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,8,8,128,1,float16,float16,0,0.025750398635864258
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,8,2,128,1,float16,float16,0,0.020550400018692017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,8,1,128,1,float16,float16,0,0.01637600064277649
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,8,4,128,1,float16,float16,0,0.018588800728321076
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,8,8,128,1,float16,float16,0,0.017726400494575502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,8,2,128,1,float16,float16,0,0.016598400473594666
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,8,2,128,1,float16,float16,0,0.015457600355148315
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,8,1,128,1,float16,float16,0,0.015273599326610566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,8,4,128,1,float16,float16,0,0.015572799742221833
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,8,1,128,1,float16,float16,0,0.01496960073709488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,8,8,128,1,float16,float16,0,0.015278400480747223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,8,2,128,1,float16,float16,0,0.014959999918937683
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,8,1,128,1,float16,float16,0,0.01478240042924881
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,8,8,128,1,float16,float16,0,0.014895999431610107
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,8,2,128,1,float16,float16,0,0.014980800449848175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,8,4,128,1,float16,float16,0,0.014444799721240997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,8,1,128,1,float16,float16,0,0.014825600385665893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,8,4,128,1,float16,float16,0,0.014977599680423736
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,8,8,128,1,float16,float16,0,0.014500799775123595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,8,2,128,1,float16,float16,0,0.014787200093269347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,8,4,128,1,float16,float16,0,0.01470080018043518
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,8,8,128,1,float16,float16,0,0.014500799775123595
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,8,1,128,1,float16,float16,0,0.014526399970054626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,8,2,128,1,float16,float16,0,0.014825600385665893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,8,4,128,1,float16,float16,0,0.014481599628925323
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,8,1,128,1,float16,float16,0,0.03129279911518097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,8,1,128,1,float16,float16,0,0.02080159932374954
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,8,8,128,1,float16,float16,0,0.05690240263938904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,8,4,128,1,float16,float16,0,0.06607199907302856
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,8,2,128,1,float16,float16,0,0.026440000534057616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,8,2,128,1,float16,float16,0,0.03908959925174713
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,8,4,128,1,float16,float16,0,0.03496800065040588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,8,8,128,1,float16,float16,0,0.03342719972133636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,8,1,128,1,float16,float16,0,0.01674720048904419
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,8,2,128,1,float16,float16,0,0.0188960000872612
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,8,8,128,1,float16,float16,0,0.02401760071516037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,8,4,128,1,float16,float16,0,0.024979199469089507
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,8,2,128,1,float16,float16,0,0.015587200224399567
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,8,1,128,1,float16,float16,0,0.01589120030403137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,8,8,128,1,float16,float16,0,0.017164799571037292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,8,4,128,1,float16,float16,0,0.01767680048942566
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,8,1,128,1,float16,float16,0,0.014918400347232819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,8,8,128,1,float16,float16,0,0.014977599680423736
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,8,4,128,1,float16,float16,0,0.014929600059986115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,8,2,128,1,float16,float16,0,0.014844800531864166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,8,1,128,1,float16,float16,0,0.014867199957370758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,8,2,128,1,float16,float16,0,0.014379200339317322
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,8,4,128,1,float16,float16,0,0.014507199823856353
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,8,8,128,1,float16,float16,0,0.014587199687957764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,8,1,128,1,float16,float16,0,0.014713600277900696
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,8,2,128,1,float16,float16,0,0.01446560025215149
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,8,4,128,1,float16,float16,0,0.014416000247001648
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,8,8,128,1,float16,float16,0,0.01422560065984726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,8,1,128,1,float16,float16,0,0.014688000082969666
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,8,2,128,1,float16,float16,0,0.014441600441932679
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,8,4,128,1,float16,float16,0,0.014326399564743042
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,8,8,128,1,float16,float16,0,0.014401599764823914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,8,1,128,1,float16,float16,0,0.014788800477981567
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,8,2,128,1,float16,float16,0,0.014393599331378936
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,8,4,128,1,float16,float16,0,0.014281600713729858
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,8,1,128,1,float16,float16,0,0.02614719867706299
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,8,2,128,1,float16,float16,0,0.03362880051136017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,8,1,128,1,float16,float16,0,0.018481600284576415
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,8,8,128,1,float16,float16,0,0.0499424010515213
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,8,2,128,1,float16,float16,0,0.02422879934310913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,8,4,128,1,float16,float16,0,0.05184000134468079
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,8,4,128,1,float16,float16,0,0.032128000259399415
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,8,8,128,1,float16,float16,0,0.030847999453544616
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,8,1,128,1,float16,float16,0,0.01568479984998703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,8,2,128,1,float16,float16,0,0.017425599694252013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,8,8,128,1,float16,float16,0,0.022815999388694764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,8,4,128,1,float16,float16,0,0.023286400735378264
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,8,1,128,1,float16,float16,0,0.014841599762439728
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,8,4,128,1,float16,float16,0,0.016784000396728515
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,8,2,128,1,float16,float16,0,0.014961600303649902
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,8,8,128,1,float16,float16,0,0.01637919992208481
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,8,2,128,1,float16,float16,0,0.014270399510860444
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,8,1,128,1,float16,float16,0,0.014289599657058717
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,8,4,128,1,float16,float16,0,0.014324800670146942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,8,8,128,1,float16,float16,0,0.014219200611114502
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,8,1,128,1,float16,float16,0,0.014009599387645722
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,8,2,128,1,float16,float16,0,0.014265599846839904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,8,4,128,1,float16,float16,0,0.0141744002699852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,8,8,128,1,float16,float16,0,0.014321599900722504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,8,1,128,1,float16,float16,0,0.014051200449466705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,8,2,128,1,float16,float16,0,0.01401119977235794
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,8,4,128,1,float16,float16,0,0.014203199744224548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,8,8,128,1,float16,float16,0,0.01427839994430542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,8,1,128,1,float16,float16,0,0.014020800590515137
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,8,2,128,1,float16,float16,0,0.014091199636459351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,8,4,128,1,float16,float16,0,0.014060799777507783
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,8,8,128,1,float16,float16,0,0.013977600634098053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,8,1,128,1,float16,float16,0,0.0137472003698349
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,8,2,128,1,float16,float16,0,0.014113600552082061
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,8,4,128,1,float16,float16,0,0.01408960074186325
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,4,4,128,1,float16,float16,0,3.087259292602539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,4,1,128,1,float16,float16,0,2.9255056381225586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,4,1,128,1,float16,float16,0,5.530120086669922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,4,2,128,1,float16,float16,0,5.619224166870117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,4,4,128,1,float16,float16,0,5.719823837280273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,4,2,128,1,float16,float16,0,3.0371679306030273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,4,4,128,1,float16,float16,0,1.7962911605834961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,4,1,128,1,float16,float16,0,1.7844144821166992
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,4,2,128,1,float16,float16,0,1.7438512802124024
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,4,4,128,1,float16,float16,0,1.8562896728515625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,4,4,128,1,float16,float16,0,3.3789710998535156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,4,1,128,1,float16,float16,0,3.2334735870361326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,4,1,128,1,float16,float16,0,10.911100769042969
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,4,2,128,1,float16,float16,0,11.085953521728516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,4,2,128,1,float16,float16,0,3.2893726348876955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,4,1,128,1,float16,float16,0,1.7536272048950194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,4,1,128,1,float16,float16,0,6.156193542480469
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,4,2,128,1,float16,float16,0,1.8680479049682617
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,4,4,128,1,float16,float16,0,1.0964608192443848
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,4,1,128,1,float16,float16,0,1.1079999923706054
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,4,2,128,1,float16,float16,0,1.137228775024414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,4,2,128,1,float16,float16,0,6.168548965454102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,4,1,128,1,float16,float16,0,1.2348112106323241
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,4,4,128,1,float16,float16,0,1.3517184257507324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,4,2,128,1,float16,float16,0,1.364948844909668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,4,4,128,1,float16,float16,0,2.3981536865234374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,4,1,128,1,float16,float16,0,2.3534576416015627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,4,2,128,1,float16,float16,0,2.3496896743774416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,4,4,128,1,float16,float16,0,0.8510623931884765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,4,1,128,1,float16,float16,0,0.8349231719970703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,4,2,128,1,float16,float16,0,0.8469247817993164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,4,1,128,1,float16,float16,0,4.373075103759765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,4,2,128,1,float16,float16,0,4.401206588745117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,4,4,128,1,float16,float16,0,1.6078912734985351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,4,1,128,1,float16,float16,0,1.58985595703125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,4,2,128,1,float16,float16,0,1.5916560173034668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,4,1,128,1,float16,float16,0,2.848084831237793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,4,2,128,1,float16,float16,0,2.8794815063476564
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,4,4,128,1,float16,float16,0,0.9395008087158203
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,4,1,128,1,float16,float16,0,0.879520034790039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,4,4,128,1,float16,float16,0,3.0136383056640623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,4,4,128,1,float16,float16,0,0.6117775917053223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,4,2,128,1,float16,float16,0,0.6156464099884034
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,4,1,128,1,float16,float16,0,0.6049407958984375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,4,2,128,1,float16,float16,0,0.9438896179199219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,4,1,128,1,float16,float16,0,5.47747688293457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,4,2,128,1,float16,float16,0,5.706131362915039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,4,1,128,1,float16,float16,0,1.645238494873047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,4,4,128,1,float16,float16,0,1.751763153076172
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,4,2,128,1,float16,float16,0,1.6835359573364257
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,4,4,128,1,float16,float16,0,1.0154128074645996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,4,1,128,1,float16,float16,0,3.1703279495239256
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,4,4,128,1,float16,float16,0,0.615774393081665
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,4,1,128,1,float16,float16,0,0.5834303855895996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,4,2,128,1,float16,float16,0,0.9566335678100586
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,4,1,128,1,float16,float16,0,1.0100576400756835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,4,1,128,1,float16,float16,0,0.3243904113769531
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,4,4,128,1,float16,float16,0,0.4001664161682129
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,4,2,128,1,float16,float16,0,0.5593264102935791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,4,2,128,1,float16,float16,0,0.39495201110839845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,4,2,128,1,float16,float16,0,3.2458271026611327
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,4,4,128,1,float16,float16,0,0.871127986907959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,4,1,128,1,float16,float16,0,0.8488224029541016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,4,4,128,1,float16,float16,0,1.5901840209960938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,4,1,128,1,float16,float16,0,1.472443199157715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,4,2,128,1,float16,float16,0,1.4869888305664063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,4,4,128,1,float16,float16,0,0.5327055931091309
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,4,2,128,1,float16,float16,0,0.8150527954101563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,4,4,128,1,float16,float16,0,0.3348896026611328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,4,1,128,1,float16,float16,0,0.33574399948120115
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,4,2,128,1,float16,float16,0,0.48282561302185056
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,4,1,128,1,float16,float16,0,0.47506399154663087
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,4,1,128,1,float16,float16,0,2.8261392593383787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,4,2,128,1,float16,float16,0,0.3099247932434082
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,4,1,128,1,float16,float16,0,0.20029280185699463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,4,2,128,1,float16,float16,0,2.9309520721435547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,4,4,128,1,float16,float16,0,0.24727199077606202
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,4,2,128,1,float16,float16,0,0.20652639865875244
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,4,1,128,1,float16,float16,0,0.49193921089172366
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,4,4,128,1,float16,float16,0,0.5480591773986816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,4,4,128,1,float16,float16,0,0.9750687599182128
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,4,1,128,1,float16,float16,0,0.8880703926086426
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,4,4,128,1,float16,float16,0,0.33408479690551757
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,4,2,128,1,float16,float16,0,0.9373104095458984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,4,2,128,1,float16,float16,0,0.5036719799041748
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,4,1,128,1,float16,float16,0,1.6369775772094726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,4,2,128,1,float16,float16,0,0.3047231912612915
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,4,1,128,1,float16,float16,0,0.29964959621429443
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,4,4,128,1,float16,float16,0,0.1894063949584961
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,4,1,128,1,float16,float16,0,0.20811679363250732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,4,2,128,1,float16,float16,0,0.19649440050125122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,4,2,128,1,float16,float16,0,1.7042655944824219
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,4,4,128,1,float16,float16,0,0.142193603515625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,4,1,128,1,float16,float16,0,0.1325824022293091
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,4,2,128,1,float16,float16,0,0.1325984001159668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,4,1,128,1,float16,float16,0,0.5307024002075196
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,4,4,128,1,float16,float16,0,0.6116127967834473
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,4,2,128,1,float16,float16,0,0.5549551963806152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,4,4,128,1,float16,float16,0,1.1645567893981934
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,4,1,128,1,float16,float16,0,1.0294032096862793
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,4,2,128,1,float16,float16,0,1.0767120361328124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,4,4,128,1,float16,float16,0,0.32349600791931155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,4,1,128,1,float16,float16,0,0.2898175954818726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,4,2,128,1,float16,float16,0,0.3016400098800659
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,4,4,128,1,float16,float16,0,0.1885167956352234
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,4,1,128,1,float16,float16,0,0.17420799732208253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,4,4,128,1,float16,float16,0,0.11341279745101929
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,4,1,128,1,float16,float16,0,0.10436160564422607
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,4,1,128,1,float16,float16,0,1.9807279586791993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,4,2,128,1,float16,float16,0,0.1805727958679199
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,4,4,128,1,float16,float16,0,0.08737279772758484
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,4,2,128,1,float16,float16,0,0.1054751992225647
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,4,1,128,1,float16,float16,0,0.07571039795875549
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,4,2,128,1,float16,float16,0,0.07527359724044799
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,4,2,128,1,float16,float16,0,2.099511909484863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,4,1,128,1,float16,float16,0,0.33625919818878175
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,4,4,128,1,float16,float16,0,0.39461441040039064
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,4,1,128,1,float16,float16,0,0.6300975799560546
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,4,2,128,1,float16,float16,0,0.3604752063751221
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,4,2,128,1,float16,float16,0,0.6650815963745117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,4,4,128,1,float16,float16,0,0.7469056129455567
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,4,1,128,1,float16,float16,0,0.19197280406951905
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,4,4,128,1,float16,float16,0,0.21915040016174317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,4,2,128,1,float16,float16,0,0.20004000663757324
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,4,4,128,1,float16,float16,0,0.13240000009536743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,4,1,128,1,float16,float16,0,0.11739200353622437
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,4,4,128,1,float16,float16,0,0.0851360023021698
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,4,1,128,1,float16,float16,0,1.2163760185241699
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,4,1,128,1,float16,float16,0,0.08071519732475281
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,4,2,128,1,float16,float16,0,0.12461279630661011
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,4,2,128,1,float16,float16,0,0.0820847988128662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,4,4,128,1,float16,float16,0,0.061268800497055055
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,4,1,128,1,float16,float16,0,0.05890719890594483
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,4,2,128,1,float16,float16,0,1.2885087966918944
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,4,2,128,1,float16,float16,0,0.05994719862937927
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,4,4,128,1,float16,float16,0,0.41276798248291013
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,4,1,128,1,float16,float16,0,0.6363167762756348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,4,2,128,1,float16,float16,0,0.3615583896636963
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,4,1,128,1,float16,float16,0,0.33765599727630613
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,4,2,128,1,float16,float16,0,0.6879263877868652
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,4,4,128,1,float16,float16,0,0.22849440574645996
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,4,4,128,1,float16,float16,0,0.7862592220306397
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,4,1,128,1,float16,float16,0,1.2356368064880372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,4,4,128,1,float16,float16,0,0.13487839698791504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,4,1,128,1,float16,float16,0,0.18956480026245118
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,4,2,128,1,float16,float16,0,0.20185279846191406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,4,1,128,1,float16,float16,0,0.11326400041580201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,4,2,128,1,float16,float16,0,0.12043520212173461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,4,2,128,1,float16,float16,0,1.3286255836486816
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,4,4,128,1,float16,float16,0,0.06883999705314636
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,4,4,128,1,float16,float16,0,0.07651039958000183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,4,1,128,1,float16,float16,0,0.0649504005908966
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,4,2,128,1,float16,float16,0,0.06637279987335205
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,4,1,128,1,float16,float16,0,0.053711998462677005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,4,2,128,1,float16,float16,0,0.06553919911384583
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,4,1,128,1,float16,float16,0,0.04334079921245575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,4,2,128,1,float16,float16,0,0.044310399889945985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,4,4,128,1,float16,float16,0,0.04429759979248047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,4,4,128,1,float16,float16,0,0.6139679908752441
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,4,1,128,1,float16,float16,0,0.4441520214080811
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,4,1,128,1,float16,float16,0,0.8572527885437011
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,4,2,128,1,float16,float16,0,0.9558143615722656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,4,1,128,1,float16,float16,0,0.23745279312133788
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,4,2,128,1,float16,float16,0,0.49243841171264646
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,4,4,128,1,float16,float16,0,0.3195440053939819
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,4,2,128,1,float16,float16,0,0.2638400077819824
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,4,4,128,1,float16,float16,0,0.1748960018157959
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,4,1,128,1,float16,float16,0,0.13394399881362914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,4,4,128,1,float16,float16,0,0.1036944031715393
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,4,2,128,1,float16,float16,0,0.146014404296875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,4,1,128,1,float16,float16,0,0.07952319979667663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,4,2,128,1,float16,float16,0,0.0887503981590271
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,4,4,128,1,float16,float16,0,0.0682911992073059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,4,1,128,1,float16,float16,0,0.05279200077056885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,4,4,128,1,float16,float16,0,0.05988640189170837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,4,1,128,1,float16,float16,0,0.052193599939346316
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,4,2,128,1,float16,float16,0,0.05433599948883057
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,4,2,128,1,float16,float16,0,0.053723198175430295
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,4,4,128,1,float16,float16,0,0.03657119870185852
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,4,1,128,1,float16,float16,0,0.03588959872722626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,4,2,128,1,float16,float16,0,0.035708799958229065
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,4,4,128,1,float16,float16,0,0.02871519923210144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,4,1,128,1,float16,float16,0,0.028411200642585753
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,4,2,128,1,float16,float16,0,0.028672000765800475
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,4,2,128,1,float16,float16,0,0.26074719429016113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,4,1,128,1,float16,float16,0,0.11912480592727662
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,4,2,128,1,float16,float16,0,0.14417439699172974
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,4,1,128,1,float16,float16,0,0.21486399173736573
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,4,4,128,1,float16,float16,0,0.2062608003616333
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,4,1,128,1,float16,float16,0,0.07216640114784241
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,4,4,128,1,float16,float16,0,0.11801120042800903
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,4,4,128,1,float16,float16,0,0.06477439999580384
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,4,2,128,1,float16,float16,0,0.08225759863853455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,4,2,128,1,float16,float16,0,0.04939199984073639
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,4,1,128,1,float16,float16,0,0.04347839951515198
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,4,4,128,1,float16,float16,0,0.03731679916381836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,4,1,128,1,float16,float16,0,0.02843039929866791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,4,4,128,1,float16,float16,0,0.025492799282073975
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,4,2,128,1,float16,float16,0,0.03221760094165802
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,4,1,128,1,float16,float16,0,0.023638400435447692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,4,2,128,1,float16,float16,0,0.023691199719905853
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,4,4,128,1,float16,float16,0,0.024265600740909575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,4,2,128,1,float16,float16,0,0.022913600504398345
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,4,1,128,1,float16,float16,0,0.022843199968338012
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,4,1,128,1,float16,float16,0,0.02259040027856827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,4,4,128,1,float16,float16,0,0.023635199666023253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,4,2,128,1,float16,float16,0,0.022708800435066224
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,4,1,128,1,float16,float16,0,0.021935999393463135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,4,4,128,1,float16,float16,0,0.023342399299144743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,4,2,128,1,float16,float16,0,0.022316800057888032
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,4,1,128,1,float16,float16,0,0.10600639581680298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,4,4,128,1,float16,float16,0,0.05973920226097107
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,4,1,128,1,float16,float16,0,0.06258559823036194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,4,1,128,1,float16,float16,0,0.03470720052719116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,4,2,128,1,float16,float16,0,0.132368004322052
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,4,2,128,1,float16,float16,0,0.0435696005821228
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,4,2,128,1,float16,float16,0,0.07324320077896118
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,4,4,128,1,float16,float16,0,0.11170239448547363
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,4,4,128,1,float16,float16,0,0.030300799012184142
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,4,2,128,1,float16,float16,0,0.025180798768997193
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,4,4,128,1,float16,float16,0,0.02067520022392273
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,4,1,128,1,float16,float16,0,0.024038399755954742
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,4,2,128,1,float16,float16,0,0.01876319944858551
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,4,4,128,1,float16,float16,0,0.01916320025920868
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,4,4,128,1,float16,float16,0,0.018771199882030486
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,4,2,128,1,float16,float16,0,0.018104000389575957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,4,1,128,1,float16,float16,0,0.01781439930200577
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,4,1,128,1,float16,float16,0,0.01794240027666092
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,4,1,128,1,float16,float16,0,0.018489600718021394
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,4,2,128,1,float16,float16,0,0.01775680035352707
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,4,4,128,1,float16,float16,0,0.018244799971580506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,4,1,128,1,float16,float16,0,0.01738400012254715
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,4,2,128,1,float16,float16,0,0.017539200186729432
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,4,4,128,1,float16,float16,0,0.017982399463653563
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,4,2,128,1,float16,float16,0,0.017369599640369417
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,4,1,128,1,float16,float16,0,0.017203199863433837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,4,1,128,1,float16,float16,0,0.06135680079460144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,4,4,128,1,float16,float16,0,0.06139519810676575
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,4,2,128,1,float16,float16,0,0.07414399981498718
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,4,1,128,1,float16,float16,0,0.020742399990558623
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,4,4,128,1,float16,float16,0,0.03115839958190918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,4,1,128,1,float16,float16,0,0.031969600915908815
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,4,2,128,1,float16,float16,0,0.04301440119743347
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,4,2,128,1,float16,float16,0,0.02131360024213791
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,4,2,128,1,float16,float16,0,0.017054399847984313
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,4,4,128,1,float16,float16,0,0.01974560022354126
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,4,1,128,1,float16,float16,0,0.015489600598812103
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,4,4,128,1,float16,float16,0,0.015801599621772765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,4,1,128,1,float16,float16,0,0.016667200624942778
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,4,2,128,1,float16,float16,0,0.015718400478363037
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,4,4,128,1,float16,float16,0,0.015193599462509155
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,4,1,128,1,float16,float16,0,0.015318399667739869
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,4,2,128,1,float16,float16,0,0.015240000188350677
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,4,4,128,1,float16,float16,0,0.014873600006103516
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,4,1,128,1,float16,float16,0,0.014920000731945039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,4,4,128,1,float16,float16,0,0.014617599546909332
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,4,2,128,1,float16,float16,0,0.014963200688362122
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,4,2,128,1,float16,float16,0,0.014864000678062438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,4,1,128,1,float16,float16,0,0.014839999377727509
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,4,4,128,1,float16,float16,0,0.014486399292945863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,4,1,128,1,float16,float16,0,0.014847999811172486
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,4,2,128,1,float16,float16,0,0.015035200119018554
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,4,2,128,1,float16,float16,0,0.049584001302719116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,4,2,128,1,float16,float16,0,0.027819201350212097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,4,4,128,1,float16,float16,0,0.03975839912891388
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,4,1,128,1,float16,float16,0,0.0330704003572464
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,4,4,128,1,float16,float16,0,0.026128000020980834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,4,1,128,1,float16,float16,0,0.021372799575328828
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,4,1,128,1,float16,float16,0,0.01740639954805374
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,4,1,128,1,float16,float16,0,0.0160288006067276
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,4,4,128,1,float16,float16,0,0.018287999927997588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,4,4,128,1,float16,float16,0,0.015304000675678253
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,4,1,128,1,float16,float16,0,0.015255999565124512
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,4,2,128,1,float16,float16,0,0.01929119974374771
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,4,2,128,1,float16,float16,0,0.014953599870204925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,4,2,128,1,float16,float16,0,0.01594880074262619
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,4,4,128,1,float16,float16,0,0.014920000731945039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,4,1,128,1,float16,float16,0,0.014830400049686433
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,4,4,128,1,float16,float16,0,0.014476799964904785
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,4,2,128,1,float16,float16,0,0.014632000029087067
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,4,2,128,1,float16,float16,0,0.014563199877738953
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,4,4,128,1,float16,float16,0,0.014412799477577209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,4,1,128,1,float16,float16,0,0.014779199659824372
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,4,1,128,1,float16,float16,0,0.014800000190734863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,4,2,128,1,float16,float16,0,0.01451359987258911
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,4,4,128,1,float16,float16,0,0.014339199662208557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,4,1,128,1,float16,float16,0,0.014822399616241455
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,4,2,128,1,float16,float16,0,0.014321599900722504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,4,2,128,1,float16,float16,0,0.03661920130252838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,4,1,128,1,float16,float16,0,0.028433600068092348
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,4,1,128,1,float16,float16,0,0.019828799366950988
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,4,4,128,1,float16,float16,0,0.03446399867534637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,4,2,128,1,float16,float16,0,0.025879999995231627
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,4,1,128,1,float16,float16,0,0.01634719967842102
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,4,4,128,1,float16,float16,0,0.024670399725437164
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,4,4,128,1,float16,float16,0,0.01764799952507019
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,4,1,128,1,float16,float16,0,0.014496000111103058
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,4,2,128,1,float16,float16,0,0.018408000469207764
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,4,1,128,1,float16,float16,0,0.015270400047302245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,4,2,128,1,float16,float16,0,0.01558080017566681
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,4,4,128,1,float16,float16,0,0.014959999918937683
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,4,2,128,1,float16,float16,0,0.01464959979057312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,4,4,128,1,float16,float16,0,0.01451520025730133
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,4,1,128,1,float16,float16,0,0.014372800290584565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,4,1,128,1,float16,float16,0,0.014401599764823914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,4,2,128,1,float16,float16,0,0.014539200067520141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,4,4,128,1,float16,float16,0,0.014431999623775482
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,4,1,128,1,float16,float16,0,0.014180800318717957
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,4,2,128,1,float16,float16,0,0.014215999841690063
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,4,4,128,1,float16,float16,0,0.014440000057220459
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,4,2,128,1,float16,float16,0,0.014372800290584565
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,4,4,128,1,float16,float16,0,0.014209599792957306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,4,1,128,1,float16,float16,0,0.014353600144386292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,4,2,128,1,float16,float16,0,0.014483200013637542
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,4,1,128,1,float16,float16,0,0.026132801175117494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,4,2,128,1,float16,float16,0,0.03378559947013855
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,4,4,128,1,float16,float16,0,0.031891199946403506
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,4,1,128,1,float16,float16,0,0.01852000057697296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,4,2,128,1,float16,float16,0,0.024395200610160827
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,4,1,128,1,float16,float16,0,0.015846399962902068
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,4,2,128,1,float16,float16,0,0.017473599314689635
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,4,4,128,1,float16,float16,0,0.023316800594329834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,4,2,128,1,float16,float16,0,0.015003199875354766
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,4,1,128,1,float16,float16,0,0.014764800667762756
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,4,4,128,1,float16,float16,0,0.01674560010433197
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,4,4,128,1,float16,float16,0,0.014339199662208557
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,4,1,128,1,float16,float16,0,0.014244799315929414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,4,4,128,1,float16,float16,0,0.014368000626564025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,4,2,128,1,float16,float16,0,0.014188799262046813
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,4,1,128,1,float16,float16,0,0.01404000073671341
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,4,2,128,1,float16,float16,0,0.014227199554443359
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,4,4,128,1,float16,float16,0,0.01422400027513504
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,4,1,128,1,float16,float16,0,0.01403679996728897
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,4,2,128,1,float16,float16,0,0.014206400513648987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,4,4,128,1,float16,float16,0,0.01401280015707016
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,4,1,128,1,float16,float16,0,0.013899199664592743
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,4,2,128,1,float16,float16,0,0.014019200205802917
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,4,4,128,1,float16,float16,0,0.014091199636459351
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,4,1,128,1,float16,float16,0,0.013736000657081604
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,4,2,128,1,float16,float16,0,0.013995200395584106
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,2,2,128,1,float16,float16,0,1.1928272247314453
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,2,1,128,1,float16,float16,0,1.6728704452514649
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,2,2,128,1,float16,float16,0,1.7492000579833984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,2,1,128,1,float16,float16,0,1.1774800300598145
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,2,1,128,1,float16,float16,0,2.9913360595703127
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,2,2,128,1,float16,float16,0,3.1063583374023436
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,2,2,128,1,float16,float16,0,1.8256912231445312
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,2,2,128,1,float16,float16,0,1.0824224472045898
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,2,1,128,1,float16,float16,0,1.1245488166809081
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,2,1,128,1,float16,float16,0,1.8240047454833985
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,2,2,128,1,float16,float16,0,0.7754511833190918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,2,1,128,1,float16,float16,0,0.626087999343872
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,12288,2,1,128,1,float16,float16,0,3.3352191925048826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16384,2,1,128,1,float16,float16,0,5.716571044921875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,2,2,128,1,float16,float16,0,0.8421119689941406
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,2,1,128,1,float16,float16,0,0.8406304359436035
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,2,2,128,1,float16,float16,0,0.6081664085388183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,2,2,128,1,float16,float16,0,1.3666303634643555
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,2,1,128,1,float16,float16,0,1.3218607902526855
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,2,1,128,1,float16,float16,0,0.5255055904388428
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,10240,2,1,128,1,float16,float16,0,2.326313591003418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,2,2,128,1,float16,float16,0,0.6107488155364991
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,2,2,128,1,float16,float16,0,0.9345088005065918
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,2,1,128,1,float16,float16,0,0.8946703910827637
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,2,2,128,1,float16,float16,0,0.41625280380249025
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,2,1,128,1,float16,float16,0,1.5662832260131836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,2,1,128,1,float16,float16,0,0.6144559860229493
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,2,1,128,1,float16,float16,0,0.36406240463256834
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,2,2,128,1,float16,float16,0,1.5903488159179688
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,2,2,128,1,float16,float16,0,0.38645761013031005
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,2,1,128,1,float16,float16,0,0.3875488042831421
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,2,1,128,1,float16,float16,0,0.5466095924377441
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,2,2,128,1,float16,float16,0,0.5995664119720459
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,2,2,128,1,float16,float16,0,0.2548127889633179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,2,1,128,1,float16,float16,0,0.24835360050201416
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,2,2,128,1,float16,float16,0,0.9707776069641113
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,2,1,128,1,float16,float16,0,0.9416416168212891
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,8192,2,1,128,1,float16,float16,0,2.850326347351074
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,6144,2,1,128,1,float16,float16,0,1.6617008209228517
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,2,2,128,1,float16,float16,0,0.19735679626464844
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,2,2,128,1,float16,float16,0,0.32315199375152587
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,2,1,128,1,float16,float16,0,0.31895520687103274
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,2,2,128,1,float16,float16,0,0.5471856117248535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,2,2,128,1,float16,float16,0,1.025823974609375
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,2,1,128,1,float16,float16,0,0.18640480041503907
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,2,1,128,1,float16,float16,0,0.5199935913085938
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,2,2,128,1,float16,float16,0,0.11818879842758179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,2,1,128,1,float16,float16,0,0.9614879608154296
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,2,1,128,1,float16,float16,0,0.11870559453964233
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,4096,2,1,128,1,float16,float16,0,1.8649904251098632
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,2,1,128,1,float16,float16,0,0.21172959804534913
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,2,1,128,1,float16,float16,0,0.33494720458984373
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,2,2,128,1,float16,float16,0,0.21487998962402344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,2,2,128,1,float16,float16,0,0.3522511959075928
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,2,2,128,1,float16,float16,0,0.15125279426574706
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,2,1,128,1,float16,float16,0,0.5994272232055664
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,2,2,128,1,float16,float16,0,0.6414959907531739
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,2,2,128,1,float16,float16,0,0.09087200164794922
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,2,1,128,1,float16,float16,0,0.08941760063171386
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,2,1,128,1,float16,float16,0,0.14572479724884033
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,3072,2,1,128,1,float16,float16,0,1.1181103706359863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,2,2,128,1,float16,float16,0,0.19863840341567993
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,2,2,128,1,float16,float16,0,0.3405695915222168
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,2,1,128,1,float16,float16,0,0.3151263952255249
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,2,1,128,1,float16,float16,0,0.19270240068435668
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,2,2,128,1,float16,float16,0,0.11435519456863404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,2,1,128,1,float16,float16,0,0.10550080537796021
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,2,2,128,1,float16,float16,0,0.05818719863891601
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,2,1,128,1,float16,float16,0,0.575219202041626
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,2,2,128,1,float16,float16,0,0.6444287776947022
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,2,2,128,1,float16,float16,0,0.0862272024154663
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,2,1,128,1,float16,float16,0,0.07602559924125671
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,2,1,128,1,float16,float16,0,0.05646560192108154
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,2048,2,1,128,1,float16,float16,0,1.0769871711730956
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,2,1,128,1,float16,float16,0,0.12955199480056762
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,2,1,128,1,float16,float16,0,0.20583200454711914
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,2,2,128,1,float16,float16,0,0.13757280111312867
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,2,2,128,1,float16,float16,0,0.22706398963928223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,2,2,128,1,float16,float16,0,0.08629599809646607
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,2,2,128,1,float16,float16,0,0.4027279853820801
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,2,1,128,1,float16,float16,0,0.3664720058441162
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,2,2,128,1,float16,float16,0,0.06050559878349304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,2,1,128,1,float16,float16,0,0.05944799780845642
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,2,1,128,1,float16,float16,0,0.08223839998245239
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,2,2,128,1,float16,float16,0,0.043663999438285826
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1536,2,1,128,1,float16,float16,0,0.6742015838623047
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,2,1,128,1,float16,float16,0,0.04279359877109527
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,2,2,128,1,float16,float16,0,0.08900160193443299
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,2,2,128,1,float16,float16,0,0.1368175983428955
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,2,1,128,1,float16,float16,0,0.2061568021774292
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,2,1,128,1,float16,float16,0,0.12408319711685181
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,2,1,128,1,float16,float16,0,0.3684799909591675
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,2,2,128,1,float16,float16,0,0.2320336103439331
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,2,1,128,1,float16,float16,0,0.07658560276031494
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,2,2,128,1,float16,float16,0,0.06949279904365539
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,2,2,128,1,float16,float16,0,0.4246816158294678
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,2,1,128,1,float16,float16,0,0.06524959802627564
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,2,2,128,1,float16,float16,0,0.04459199905395508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1024,2,1,128,1,float16,float16,0,0.6998415946960449
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,2,1,128,1,float16,float16,0,0.03327839970588684
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,2,1,128,1,float16,float16,0,0.04391840100288391
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,2,2,128,1,float16,float16,0,0.0332751989364624
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,2,2,128,1,float16,float16,0,0.1041983962059021
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,2,1,128,1,float16,float16,0,0.08912960290908814
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,2,1,128,1,float16,float16,0,0.14832799434661864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,2,2,128,1,float16,float16,0,0.18032799959182738
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,2,1,128,1,float16,float16,0,0.26557919979095457
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,2,2,128,1,float16,float16,0,0.32776000499725344
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,2,2,128,1,float16,float16,0,0.06766240000724792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,2,1,128,1,float16,float16,0,0.055343997478485105
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,512,2,1,128,1,float16,float16,0,0.5044256210327148
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,2,2,128,1,float16,float16,0,0.060222399234771726
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,2,1,128,1,float16,float16,0,0.05280640125274658
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,2,2,128,1,float16,float16,0,0.036329600214958194
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,2,1,128,1,float16,float16,0,0.026932799816131593
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,2,2,128,1,float16,float16,0,0.028779199719429015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,2,1,128,1,float16,float16,0,0.028590399026870727
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,2,1,128,1,float16,float16,0,0.036052799224853514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,2,2,128,1,float16,float16,0,0.027081599831581114
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,2,1,128,1,float16,float16,0,0.049377599358558656
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,2,1,128,1,float16,float16,0,0.08308159708976745
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,2,2,128,1,float16,float16,0,0.0656336009502411
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,2,2,128,1,float16,float16,0,0.03854399919509888
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,256,2,1,128,1,float16,float16,0,0.1460479974746704
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,2,2,128,1,float16,float16,0,0.02574560046195984
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,2,2,128,1,float16,float16,0,0.12028640508651733
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,2,1,128,1,float16,float16,0,0.033497598767280576
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,2,1,128,1,float16,float16,0,0.02382880002260208
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,2,1,128,1,float16,float16,0,0.023080000281333925
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,2,2,128,1,float16,float16,0,0.024291199445724488
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,2,1,128,1,float16,float16,0,0.022329600155353548
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,2,1,128,1,float16,float16,0,0.02279680073261261
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,2,2,128,1,float16,float16,0,0.023827199637889863
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,2,2,128,1,float16,float16,0,0.02309119999408722
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,2,2,128,1,float16,float16,0,0.02327200025320053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,2,1,128,1,float16,float16,0,0.022195200622081756
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,2,2,128,1,float16,float16,0,0.06125440001487732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,2,1,128,1,float16,float16,0,0.04436799883842468
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,2,1,128,1,float16,float16,0,0.02569440007209778
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,2,2,128,1,float16,float16,0,0.030406400561332703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,2,2,128,1,float16,float16,0,0.021252800524234772
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,2,1,128,1,float16,float16,0,0.019729599356651306
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,128,2,1,128,1,float16,float16,0,0.07557920217514039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,2,2,128,1,float16,float16,0,0.019215999543666838
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,2,1,128,1,float16,float16,0,0.01833920031785965
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,2,2,128,1,float16,float16,0,0.01857759952545166
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,2,2,128,1,float16,float16,0,0.018225599825382233
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,2,1,128,1,float16,float16,0,0.017977599799633027
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,2,1,128,1,float16,float16,0,0.017582400143146514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,2,2,128,1,float16,float16,0,0.01804320067167282
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,2,1,128,1,float16,float16,0,0.017350399494171144
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,2,2,128,1,float16,float16,0,0.018129600584506987
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,2,1,128,1,float16,float16,0,0.017342400550842286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,64,2,1,128,1,float16,float16,0,0.04335840046405792
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,2,2,128,1,float16,float16,0,0.03171359896659851
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,2,1,128,1,float16,float16,0,0.02279680073261261
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,2,2,128,1,float16,float16,0,0.020425599813461304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,2,1,128,1,float16,float16,0,0.017811200022697447
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,2,2,128,1,float16,float16,0,0.01660960018634796
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,2,1,128,1,float16,float16,0,0.01635199934244156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,2,2,128,1,float16,float16,0,0.015267199277877808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,2,1,128,1,float16,float16,0,0.015022400021553039
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,2,2,128,1,float16,float16,0,0.014856000244617463
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,2,1,128,1,float16,float16,0,0.015438400208950043
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,2,1,128,1,float16,float16,0,0.015515199303627015
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,2,2,128,1,float16,float16,0,0.014455999433994293
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,2,2,128,1,float16,float16,0,0.014593599736690522
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,2,1,128,1,float16,float16,0,0.014638400077819825
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,2,2,128,1,float16,float16,0,0.014459200203418732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,2,1,128,1,float16,float16,0,0.01462559998035431
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,32,2,1,128,1,float16,float16,0,0.030003198981285097
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,2,1,128,1,float16,float16,0,0.02051199972629547
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,2,2,128,1,float16,float16,0,0.02733280062675476
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,2,2,128,1,float16,float16,0,0.015827199816703795
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,2,1,128,1,float16,float16,0,0.015703999996185304
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,2,2,128,1,float16,float16,0,0.018852800130844116
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,2,1,128,1,float16,float16,0,0.01656319946050644
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,2,2,128,1,float16,float16,0,0.015000000596046448
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,2,1,128,1,float16,float16,0,0.014689600467681885
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,2,1,128,1,float16,float16,0,0.014457599818706512
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,2,2,128,1,float16,float16,0,0.014590400457382201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,2,2,128,1,float16,float16,0,0.01430719941854477
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,2,1,128,1,float16,float16,0,0.01446239948272705
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,2,2,128,1,float16,float16,0,0.014259199798107147
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,2,1,128,1,float16,float16,0,0.014424000680446625
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,2,2,128,1,float16,float16,0,0.014324800670146942
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,2,1,128,1,float16,float16,0,0.014343999326229095
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,16,2,1,128,1,float16,float16,0,0.028009599447250365
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,2,2,128,1,float16,float16,0,0.018137599527835845
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,2,1,128,1,float16,float16,0,0.019489599764347075
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,2,1,128,1,float16,float16,0,0.016174399852752687
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,2,2,128,1,float16,float16,0,0.025756800174713136
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,2,1,128,1,float16,float16,0,0.015252800285816192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,2,2,128,1,float16,float16,0,0.015414400398731232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,2,2,128,1,float16,float16,0,0.01459999978542328
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,2,1,128,1,float16,float16,0,0.014232000708580017
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,2,2,128,1,float16,float16,0,0.014542399346828461
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,2,1,128,1,float16,float16,0,0.014297600090503692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,2,1,128,1,float16,float16,0,0.014297600090503692
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,2,2,128,1,float16,float16,0,0.014108799397945404
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,2,1,128,1,float16,float16,0,0.014327999949455262
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,2,2,128,1,float16,float16,0,0.014265599846839904
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,2,2,128,1,float16,float16,0,0.014254400134086609
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,2,1,128,1,float16,float16,0,0.01395999938249588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,2,2,128,1,float16,float16,0,0.024414399266242982
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,2,1,128,1,float16,float16,0,0.018598400056362152
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,2,2,128,1,float16,float16,0,0.01751199960708618
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,2,1,128,1,float16,float16,0,0.01579679995775223
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,256,1,2,1,128,1,float16,float16,0,0.026233598589897156
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,2,2,128,1,float16,float16,0,0.01493919938802719
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,2,1,128,1,float16,float16,0,0.014820800721645355
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,2,2,128,1,float16,float16,0,0.014302399754524232
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,2,1,128,1,float16,float16,0,0.014008000493049622
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,2,2,128,1,float16,float16,0,0.014313599467277527
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,2,2,128,1,float16,float16,0,0.014001600444316864
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,2,1,128,1,float16,float16,0,0.013935999572277069
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,2,2,128,1,float16,float16,0,0.014144000411033631
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,2,1,128,1,float16,float16,0,0.013945600390434265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,2,1,128,1,float16,float16,0,0.013697600364685059
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,2,2,128,1,float16,float16,0,0.014105600118637086
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,2,1,128,1,float16,float16,0,0.013755199313163758
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,12288,1,1,128,1,float16,float16,0,0.4844560146331787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16384,1,1,128,1,float16,float16,0,0.7077839851379395
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,12288,1,1,128,1,float16,float16,0,0.769323205947876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,12288,1,1,128,1,float16,float16,0,1.0671792030334473
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16384,1,1,128,1,float16,float16,0,1.1994463920593261
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,10240,1,1,128,1,float16,float16,0,0.23920960426330568
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,10240,1,1,128,1,float16,float16,0,0.4317615985870361
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,10240,1,1,128,1,float16,float16,0,0.7823791980743409
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16384,1,1,128,1,float16,float16,0,1.756879997253418
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,8192,1,1,128,1,float16,float16,0,0.3506495952606201
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,8192,1,1,128,1,float16,float16,0,0.19566559791564941
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,8192,1,1,128,1,float16,float16,0,0.5850240230560303
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,6144,1,1,128,1,float16,float16,0,0.15072959661483765
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,6144,1,1,128,1,float16,float16,0,0.21761438846588135
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,6144,1,1,128,1,float16,float16,0,0.37556641101837157
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,8192,1,1,128,1,float16,float16,0,0.9910479545593261
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,4096,1,1,128,1,float16,float16,0,0.19127520322799682
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,6144,1,1,128,1,float16,float16,0,0.6184783935546875
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,4096,1,1,128,1,float16,float16,0,0.11737439632415772
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,4096,1,1,128,1,float16,float16,0,0.08307679891586303
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,4096,1,1,128,1,float16,float16,0,0.33627679347991946
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,4096,1,1,128,1,float16,float16,0,0.5688447952270508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,3072,1,1,128,1,float16,float16,0,0.14484959840774536
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,3072,1,1,128,1,float16,float16,0,0.05653759837150574
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,3072,1,1,128,1,float16,float16,0,0.22715840339660645
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,3072,1,1,128,1,float16,float16,0,0.09115520119667053
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,3072,1,1,128,1,float16,float16,0,0.366809606552124
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,2048,1,1,128,1,float16,float16,0,0.2078847885131836
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,2048,1,1,128,1,float16,float16,0,0.12770240306854247
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,2048,1,1,128,1,float16,float16,0,0.04227040112018585
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,2048,1,1,128,1,float16,float16,0,0.08631839752197265
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,2048,1,1,128,1,float16,float16,0,0.05734879970550537
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,2048,1,1,128,1,float16,float16,0,0.3476560115814209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1536,1,1,128,1,float16,float16,0,0.09144960045814514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1536,1,1,128,1,float16,float16,0,0.14823999404907226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1536,1,1,128,1,float16,float16,0,0.043191999197006226
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1536,1,1,128,1,float16,float16,0,0.06116799712181091
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1536,1,1,128,1,float16,float16,0,0.03320800065994263
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1536,1,1,128,1,float16,float16,0,0.23078560829162598
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1024,1,1,128,1,float16,float16,0,0.03330399990081787
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1024,1,1,128,1,float16,float16,0,0.0685375988483429
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1024,1,1,128,1,float16,float16,0,0.044547200202941895
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1024,1,1,128,1,float16,float16,0,0.2375920057296753
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1024,1,1,128,1,float16,float16,0,0.0881488025188446
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1024,1,1,128,1,float16,float16,0,0.13820960521697997
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1024,1,1,128,1,float16,float16,0,0.03202719986438751
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,512,1,1,128,1,float16,float16,0,0.10587040185928345
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,512,1,1,128,1,float16,float16,0,0.06732640266418458
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,512,1,1,128,1,float16,float16,0,0.02736639976501465
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,512,1,1,128,1,float16,float16,0,0.059913599491119386
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,512,1,1,128,1,float16,float16,0,0.028883200883865357
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,512,1,1,128,1,float16,float16,0,0.18022719621658326
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,512,1,1,128,1,float16,float16,0,0.03628000020980835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,512,1,1,128,1,float16,float16,0,0.026150399446487428
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,256,1,1,128,1,float16,float16,0,0.0665008008480072
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,256,1,1,128,1,float16,float16,0,0.024633599817752837
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,256,1,1,128,1,float16,float16,0,0.026600000262260438
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,256,1,1,128,1,float16,float16,0,0.03860479891300202
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,256,1,1,128,1,float16,float16,0,0.023081600666046143
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,256,1,1,128,1,float16,float16,0,0.02341119945049286
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,256,1,1,128,1,float16,float16,0,0.023809599876403808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,256,1,1,128,1,float16,float16,0,0.023107199370861052
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,128,1,1,128,1,float16,float16,0,0.022251200675964356
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,128,1,1,128,1,float16,float16,0,0.031118398904800414
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,128,1,1,128,1,float16,float16,0,0.020107200741767882
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,128,1,1,128,1,float16,float16,0,0.01863040030002594
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,128,1,1,128,1,float16,float16,0,0.018248000741004945
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,128,1,1,128,1,float16,float16,0,0.01786080002784729
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,128,1,1,128,1,float16,float16,0,0.01785759925842285
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,64,1,1,128,1,float16,float16,0,0.015779200196266174
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,128,1,1,128,1,float16,float16,0,0.017827199399471284
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,64,1,1,128,1,float16,float16,0,0.021799999475479125
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,64,1,1,128,1,float16,float16,0,0.017548799514770508
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,64,1,1,128,1,float16,float16,0,0.014659200608730317
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,64,1,1,128,1,float16,float16,0,0.01496479958295822
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,64,1,1,128,1,float16,float16,0,0.014364799857139588
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,64,1,1,128,1,float16,float16,0,0.014459200203418732
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,64,1,1,128,1,float16,float16,0,0.014521600306034088
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,32,1,1,128,1,float16,float16,0,0.01644960045814514
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,32,1,1,128,1,float16,float16,0,0.015452800691127777
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,32,1,1,128,1,float16,float16,0,0.014539200067520141
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,32,1,1,128,1,float16,float16,0,0.020185600221157073
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,32,1,1,128,1,float16,float16,0,0.014412799477577209
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,32,1,1,128,1,float16,float16,0,0.014235199987888336
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,32,1,1,128,1,float16,float16,0,0.014468799531459808
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,32,1,1,128,1,float16,float16,0,0.014140799641609192
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,16,1,1,128,1,float16,float16,0,0.015004800260066986
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,16,1,1,128,1,float16,float16,0,0.0144896000623703
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,16,1,1,128,1,float16,float16,0,0.019131200015544893
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,16,1,1,128,1,float16,float16,0,0.016246399283409117
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,16,1,1,128,1,float16,float16,0,0.014347200095653535
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,16,1,1,128,1,float16,float16,0,0.01417279988527298
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,16,1,1,128,1,float16,float16,0,0.01419679969549179
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,16,1,1,128,1,float16,float16,0,0.014251199364662171
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,64,1,1,1,128,1,float16,float16,0,0.01566559970378876
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,32,1,1,1,128,1,float16,float16,0,0.014803199470043183
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,16,1,1,1,128,1,float16,float16,0,0.014158399403095245
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,128,1,1,1,128,1,float16,float16,0,0.01839679926633835
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,8,1,1,1,128,1,float16,float16,0,0.014056000113487243
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,4,1,1,1,128,1,float16,float16,0,0.013952000439167023
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,2,1,1,1,128,1,float16,float16,0,0.013886399567127228
SGLang,0.5.8.post1,NVIDIA A100-SXM4-80GB,context_attention,flash_attention,1,1,1,1,128,1,float16,float16,0,0.014150400459766389
